From 82ec59427780d1090038ca83988fddb62f3d4ded Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Tue, 21 Apr 2026 21:06:48 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=AF=E7=94=A8=E6=A6=82=E7=8E=87=E6=A0=A1?= =?UTF-8?q?=E5=87=86=E5=92=8C=20WebSocket=20=E6=8A=A5=E4=BB=B7=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 5 + .../auto_retrain_report.json | 32 +- .../decision_report.json | 62 +++ .../emos-auto-20260421115828/default.json | 115 +++++ .../evaluation_report.json | 293 +++++++++++ .../decision_report.json | 21 + .../decision_report.json | 62 +++ .../emos-auto-20260421121920/default.json | 361 +++++++++++++ .../evaluation_report.json | 482 ++++++++++++++++++ .../decision_report.json | 62 +++ .../emos-auto-20260421122743/default.json | 361 +++++++++++++ .../evaluation_report.json | 482 ++++++++++++++++++ .../default.backup-20260421122808.json | 115 +++++ .../probability_calibration/default.json | 388 +++++++++++--- .../components/dashboard/Dashboard.module.css | 75 +++ .../components/dashboard/PanelSections.tsx | 112 ++++ frontend/lib/dashboard-types.ts | 34 ++ requirements.txt | 1 + src/data_collection/polymarket_readonly.py | 146 +++++- src/data_collection/polymarket_ws_cache.py | 404 +++++++++++++++ tests/test_polymarket_readonly.py | 56 ++ tests/test_polymarket_ws_cache.py | 124 +++++ 22 files changed, 3703 insertions(+), 90 deletions(-) create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/decision_report.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/default.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/evaluation_report.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121159/decision_report.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/decision_report.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/default.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/evaluation_report.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/decision_report.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/default.json create mode 100644 artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/evaluation_report.json create mode 100644 artifacts/probability_calibration/default.backup-20260421122808.json create mode 100644 src/data_collection/polymarket_ws_cache.py create mode 100644 tests/test_polymarket_ws_cache.py diff --git a/.env.example b/.env.example index 98e226cf..080a369b 100644 --- a/.env.example +++ b/.env.example @@ -177,6 +177,11 @@ POLYMARKET_CHAIN_ID=137 POLYMARKET_HTTP_TIMEOUT_SEC=8 POLYMARKET_MARKET_CACHE_TTL_SEC=180 POLYMARKET_PRICE_CACHE_TTL_SEC=10 +POLYMARKET_WS_PRICE_ENABLED=false +POLYMARKET_WS_MARKET_URL=wss://ws-subscriptions-clob.polymarket.com/ws/market +POLYMARKET_WS_QUOTE_TTL_SEC=8 +POLYMARKET_WS_MAX_ASSETS=256 +POLYMARKET_WS_RECONNECT_DELAY_SEC=3 POLYMARKET_DISCOVERY_PAGES=6 POLYMARKET_DISCOVERY_LIMIT=200 POLYMARKET_SIGNAL_MIN_LIQUIDITY=500 diff --git a/artifacts/local_runtime/probability_calibration/auto_retrain_report.json b/artifacts/local_runtime/probability_calibration/auto_retrain_report.json index c24596ed..a90686bc 100644 --- a/artifacts/local_runtime/probability_calibration/auto_retrain_report.json +++ b/artifacts/local_runtime/probability_calibration/auto_retrain_report.json @@ -1,14 +1,14 @@ { "ok": true, - "version": "emos-auto-20260418212046", - "generated_at": "2026-04-18T21:21:04.160748+00:00", - "candidate_dir": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260418212046", - "candidate_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260418212046\\default.json", - "evaluation_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260418212046\\evaluation_report.json", + "version": "emos-auto-20260421122743", + "generated_at": "2026-04-21T12:28:08.021388+00:00", + "candidate_dir": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743", + "candidate_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json", + "evaluation_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\evaluation_report.json", "target_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\default.json", - "promote_requested": false, - "promoted": false, - "backup_path": null, + "promote_requested": true, + "promoted": true, + "backup_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\default.backup-20260421122808.json", "decision": { "decision": "promote", "ready_for_promotion": true, @@ -20,10 +20,10 @@ "min_delta_bucket_hit_rate": -0.05 }, "metrics": { - "sample_count": 847, - "delta_crps": -0.03617, - "delta_mae": -0.007896, - "delta_bucket_hit_rate": -0.009445 + "sample_count": 978, + "delta_crps": -0.05721, + "delta_mae": -0.034568, + "delta_bucket_hit_rate": -0.00409 } }, "fit": { @@ -31,9 +31,9 @@ "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", "scripts/fit_probability_calibration.py", "--output", - "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260418212046\\default.json", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json", "--version", - "emos-auto-20260418212046", + "emos-auto-20260421122743", "--verbose", "--snapshot-limit", "50000" @@ -47,9 +47,9 @@ "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", "scripts/evaluate_probability_calibration.py", "--calibration-file", - "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260418212046\\default.json", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json", "--output", - "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260418212046\\evaluation_report.json", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\evaluation_report.json", "--verbose", "--snapshot-limit", "50000" diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/decision_report.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/decision_report.json new file mode 100644 index 00000000..240dea22 --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/decision_report.json @@ -0,0 +1,62 @@ +{ + "ok": true, + "version": "emos-auto-20260421115828", + "generated_at": "2026-04-21T11:58:32.490537+00:00", + "candidate_dir": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421115828", + "candidate_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421115828\\default.json", + "evaluation_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421115828\\evaluation_report.json", + "target_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\default.json", + "promote_requested": false, + "promoted": false, + "backup_path": null, + "decision": { + "decision": "promote", + "ready_for_promotion": true, + "blocking_reasons": [], + "thresholds": { + "min_samples": 50, + "max_delta_crps": 0.0, + "max_delta_mae": 0.05, + "min_delta_bucket_hit_rate": -0.05 + }, + "metrics": { + "sample_count": 74, + "delta_crps": -0.142868, + "delta_mae": -0.056741, + "delta_bucket_hit_rate": 0.0 + } + }, + "fit": { + "command": [ + "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + "scripts/fit_probability_calibration.py", + "--output", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421115828\\default.json", + "--version", + "emos-auto-20260421115828", + "--verbose", + "--snapshot-limit", + "50000" + ], + "returncode": 0, + "stdout": "", + "stderr": "" + }, + "evaluate": { + "command": [ + "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + "scripts/evaluate_probability_calibration.py", + "--calibration-file", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421115828\\default.json", + "--output", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421115828\\evaluation_report.json", + "--verbose", + "--snapshot-limit", + "50000" + ], + "returncode": 0, + "stdout": "", + "stderr": "" + }, + "tests": null +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/default.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/default.json new file mode 100644 index 00000000..c1ce49df --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/default.json @@ -0,0 +1,115 @@ +{ + "version": "emos-auto-20260421115828", + "trained_at": "2026-04-21T11:58:31.424860+00:00", + "global": { + "mu": { + "intercept": 2.60074353, + "raw_mu_coef": 1.97765258, + "deb_coef": -1.37457626, + "ens_median_coef": 0.10693472, + "max_so_far_gap_coef": -0.58256593 + }, + "sigma": { + "intercept": 0.6373609, + "raw_sigma_coef": -1.41537225, + "spread_coef": 2.12997577, + "peak_flag_coef": -0.08633203, + "max_so_far_gap_coef": -0.00716383 + } + }, + "sigma_constraints": { + "min_ratio": 0.85, + "max_ratio": 1.35, + "absolute_min": 0.25, + "absolute_max": 3.0 + }, + "selection_guardrails": { + "max_mae_increase": 0.02, + "max_bucket_hit_drop": 0.01, + "max_bucket_brier_increase": 0.05 + }, + "blending": { + "alpha_mu": 0.05, + "alpha_sigma": 0.7 + }, + "cities": { + "tokyo": { + "samples": 5, + "mu_bias": 2.101101, + "sigma_scale": 2.0, + "confidence": 0.625 + }, + "ankara": { + "samples": 6, + "mu_bias": -0.523145, + "sigma_scale": 1.139849, + "confidence": 0.75 + }, + "istanbul": { + "samples": 3, + "mu_bias": -0.285183, + "sigma_scale": 0.550171, + "confidence": 0.375 + }, + "hong kong": { + "samples": 7, + "mu_bias": 3.234676, + "sigma_scale": 2.0, + "confidence": 0.875 + }, + "taipei": { + "samples": 5, + "mu_bias": 4.731246, + "sigma_scale": 2.0, + "confidence": 0.625 + }, + "busan": { + "samples": 3, + "mu_bias": 0.177772, + "sigma_scale": 2.0, + "confidence": 0.375 + }, + "seoul": { + "samples": 3, + "mu_bias": -3.070325, + "sigma_scale": 1.462196, + "confidence": 0.375 + }, + "milan": { + "samples": 3, + "mu_bias": -4.249866, + "sigma_scale": 2.0, + "confidence": 0.375 + }, + "shanghai": { + "samples": 3, + "mu_bias": 1.080039, + "sigma_scale": 1.638119, + "confidence": 0.375 + }, + "warsaw": { + "samples": 3, + "mu_bias": -1.68611, + "sigma_scale": 1.38114, + "confidence": 0.375 + } + }, + "metrics": { + "sample_count": 74, + "mean_crps": 3.181501, + "legacy_mean_crps": 3.474108, + "legacy_mean_mae": 3.679324, + "legacy_bucket_hit_rate": 0.5, + "legacy_bucket_brier": 0.855395, + "selected_mean_crps": 3.322469, + "selected_mean_mae": 3.618254, + "selected_bucket_hit_rate": 0.5, + "selected_bucket_brier": 0.863048, + "selected_score": 5.08158, + "legacy_score": 5.233586, + "filled_actual_from_history": 0, + "settlement_history_city_count": 30, + "legacy_archive_samples": 54 + }, + "source": "artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421115828\\default.json" +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/evaluation_report.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/evaluation_report.json new file mode 100644 index 00000000..f574035d --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421115828/evaluation_report.json @@ -0,0 +1,293 @@ +{ + "summary": { + "sample_count": 74, + "filled_actual_from_history": 0, + "legacy": { + "mean_crps": 3.474108, + "mean_mae": 3.679324, + "bucket_hit_rate": 0.5 + }, + "emos": { + "mean_crps": 3.33124, + "mean_mae": 3.622584, + "bucket_hit_rate": 0.5 + }, + "delta": { + "crps": -0.142868, + "mae": -0.056741, + "bucket_hit_rate": 0.0 + } + }, + "by_city": { + "ankara": { + "samples": 6, + "legacy_mean_crps": 1.365135, + "emos_mean_crps": 1.392829, + "legacy_mean_mae": 1.466667, + "emos_mean_mae": 1.444819, + "legacy_bucket_hit_rate": 0.666667, + "emos_bucket_hit_rate": 0.666667 + }, + "atlanta": { + "samples": 2, + "legacy_mean_crps": 30.449382, + "emos_mean_crps": 29.886236, + "legacy_mean_mae": 32.015, + "emos_mean_mae": 31.322804, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "buenos aires": { + "samples": 2, + "legacy_mean_crps": 9.113412, + "emos_mean_crps": 8.463522, + "legacy_mean_mae": 10.27, + "emos_mean_mae": 10.024533, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "busan": { + "samples": 3, + "legacy_mean_crps": 0.276202, + "emos_mean_crps": 0.326485, + "legacy_mean_mae": 0.3, + "emos_mean_mae": 0.239636, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "chengdu": { + "samples": 1, + "legacy_mean_crps": 0.306469, + "emos_mean_crps": 0.378812, + "legacy_mean_mae": 0.3, + "emos_mean_mae": 0.153224, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "chicago": { + "samples": 1, + "legacy_mean_crps": 1.250268, + "emos_mean_crps": 0.714669, + "legacy_mean_mae": 0.0, + "emos_mean_mae": 0.319756, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "dallas": { + "samples": 1, + "legacy_mean_crps": 2.173363, + "emos_mean_crps": 0.840297, + "legacy_mean_mae": 0.0, + "emos_mean_mae": 1.028146, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.0 + }, + "hong kong": { + "samples": 7, + "legacy_mean_crps": 0.706099, + "emos_mean_crps": 0.731926, + "legacy_mean_mae": 0.657143, + "emos_mean_mae": 0.669911, + "legacy_bucket_hit_rate": 0.714286, + "emos_bucket_hit_rate": 0.571429 + }, + "istanbul": { + "samples": 3, + "legacy_mean_crps": 1.264103, + "emos_mean_crps": 1.214967, + "legacy_mean_mae": 1.5, + "emos_mean_mae": 1.469837, + "legacy_bucket_hit_rate": 0.333333, + "emos_bucket_hit_rate": 0.666667 + }, + "london": { + "samples": 2, + "legacy_mean_crps": 3.885033, + "emos_mean_crps": 3.799841, + "legacy_mean_mae": 4.135, + "emos_mean_mae": 4.027503, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.5 + }, + "lucknow": { + "samples": 2, + "legacy_mean_crps": 2.487193, + "emos_mean_crps": 2.107441, + "legacy_mean_mae": 3.205, + "emos_mean_mae": 2.876943, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "madrid": { + "samples": 2, + "legacy_mean_crps": 6.27726, + "emos_mean_crps": 5.810228, + "legacy_mean_mae": 7.33, + "emos_mean_mae": 7.195133, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "miami": { + "samples": 2, + "legacy_mean_crps": 28.637631, + "emos_mean_crps": 27.592664, + "legacy_mean_mae": 30.175, + "emos_mean_mae": 29.284979, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "milan": { + "samples": 3, + "legacy_mean_crps": 4.401392, + "emos_mean_crps": 3.782613, + "legacy_mean_mae": 4.06, + "emos_mean_mae": 3.989808, + "legacy_bucket_hit_rate": 0.666667, + "emos_bucket_hit_rate": 0.666667 + }, + "munich": { + "samples": 2, + "legacy_mean_crps": 3.145192, + "emos_mean_crps": 2.990712, + "legacy_mean_mae": 3.64, + "emos_mean_mae": 3.61502, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "new york": { + "samples": 1, + "legacy_mean_crps": 3.692845, + "emos_mean_crps": 3.021798, + "legacy_mean_mae": 4.94, + "emos_mean_mae": 4.54437, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "paris": { + "samples": 2, + "legacy_mean_crps": 4.013782, + "emos_mean_crps": 3.915303, + "legacy_mean_mae": 4.265, + "emos_mean_mae": 4.179694, + "legacy_bucket_hit_rate": 0.5, + "emos_bucket_hit_rate": 0.5 + }, + "sao paulo": { + "samples": 2, + "legacy_mean_crps": 5.540967, + "emos_mean_crps": 5.018605, + "legacy_mean_mae": 6.57, + "emos_mean_mae": 6.314389, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "seattle": { + "samples": 1, + "legacy_mean_crps": 0.315488, + "emos_mean_crps": 0.524119, + "legacy_mean_mae": 0.0, + "emos_mean_mae": 0.673609, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.0 + }, + "seoul": { + "samples": 3, + "legacy_mean_crps": 0.508331, + "emos_mean_crps": 0.520762, + "legacy_mean_mae": 0.2, + "emos_mean_mae": 0.247862, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "shanghai": { + "samples": 3, + "legacy_mean_crps": 0.299116, + "emos_mean_crps": 0.402453, + "legacy_mean_mae": 0.1, + "emos_mean_mae": 0.189437, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "shenzhen": { + "samples": 1, + "legacy_mean_crps": 1.198351, + "emos_mean_crps": 1.490963, + "legacy_mean_mae": 1.3, + "emos_mean_mae": 1.628189, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "singapore": { + "samples": 2, + "legacy_mean_crps": 0.281993, + "emos_mean_crps": 0.370374, + "legacy_mean_mae": 0.15, + "emos_mean_mae": 0.118005, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "taipei": { + "samples": 5, + "legacy_mean_crps": 0.950739, + "emos_mean_crps": 1.042332, + "legacy_mean_mae": 0.94, + "emos_mean_mae": 0.951241, + "legacy_bucket_hit_rate": 0.6, + "emos_bucket_hit_rate": 0.6 + }, + "tel aviv": { + "samples": 2, + "legacy_mean_crps": 0.446758, + "emos_mean_crps": 0.578691, + "legacy_mean_mae": 0.3, + "emos_mean_mae": 0.116966, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "tokyo": { + "samples": 5, + "legacy_mean_crps": 0.879366, + "emos_mean_crps": 0.876608, + "legacy_mean_mae": 1.022, + "emos_mean_mae": 1.008317, + "legacy_bucket_hit_rate": 0.2, + "emos_bucket_hit_rate": 0.4 + }, + "toronto": { + "samples": 2, + "legacy_mean_crps": 5.497916, + "emos_mean_crps": 5.268783, + "legacy_mean_mae": 6.33, + "emos_mean_mae": 6.388522, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "warsaw": { + "samples": 3, + "legacy_mean_crps": 1.618875, + "emos_mean_crps": 1.524219, + "legacy_mean_mae": 2.056667, + "emos_mean_mae": 2.006524, + "legacy_bucket_hit_rate": 0.333333, + "emos_bucket_hit_rate": 0.333333 + }, + "wellington": { + "samples": 2, + "legacy_mean_crps": 0.364919, + "emos_mean_crps": 0.484124, + "legacy_mean_mae": 0.15, + "emos_mean_mae": 0.123335, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "wuhan": { + "samples": 1, + "legacy_mean_crps": 0.476225, + "emos_mean_crps": 0.578405, + "legacy_mean_mae": 0.8, + "emos_mean_mae": 0.962852, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + } + } +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121159/decision_report.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121159/decision_report.json new file mode 100644 index 00000000..d14bccb7 --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121159/decision_report.json @@ -0,0 +1,21 @@ +{ + "ok": false, + "version": "emos-auto-20260421121159", + "stage": "fit", + "fit": { + "command": [ + "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + "scripts/fit_probability_calibration.py", + "--output", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121159\\default.json", + "--version", + "emos-auto-20260421121159", + "--verbose", + "--snapshot-limit", + "50000" + ], + "returncode": 1, + "stdout": "", + "stderr": "" + } +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/decision_report.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/decision_report.json new file mode 100644 index 00000000..0b42484c --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/decision_report.json @@ -0,0 +1,62 @@ +{ + "ok": true, + "version": "emos-auto-20260421121920", + "generated_at": "2026-04-21T12:19:41.596944+00:00", + "candidate_dir": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121920", + "candidate_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121920\\default.json", + "evaluation_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121920\\evaluation_report.json", + "target_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\default.json", + "promote_requested": false, + "promoted": false, + "backup_path": null, + "decision": { + "decision": "promote", + "ready_for_promotion": true, + "blocking_reasons": [], + "thresholds": { + "min_samples": 50, + "max_delta_crps": 0.0, + "max_delta_mae": 0.05, + "min_delta_bucket_hit_rate": -0.05 + }, + "metrics": { + "sample_count": 978, + "delta_crps": -0.05721, + "delta_mae": -0.034568, + "delta_bucket_hit_rate": -0.00409 + } + }, + "fit": { + "command": [ + "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + "scripts/fit_probability_calibration.py", + "--output", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121920\\default.json", + "--version", + "emos-auto-20260421121920", + "--verbose", + "--snapshot-limit", + "50000" + ], + "returncode": 0, + "stdout": "", + "stderr": "" + }, + "evaluate": { + "command": [ + "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + "scripts/evaluate_probability_calibration.py", + "--calibration-file", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121920\\default.json", + "--output", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121920\\evaluation_report.json", + "--verbose", + "--snapshot-limit", + "50000" + ], + "returncode": 0, + "stdout": "", + "stderr": "" + }, + "tests": null +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/default.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/default.json new file mode 100644 index 00000000..0e881615 --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/default.json @@ -0,0 +1,361 @@ +{ + "version": "emos-auto-20260421121920", + "trained_at": "2026-04-21T12:19:38.887631+00:00", + "global": { + "mu": { + "intercept": 0.34330438, + "raw_mu_coef": 0.48749629, + "deb_coef": 0.45733479, + "ens_median_coef": 0.05955297, + "max_so_far_gap_coef": -0.104935 + }, + "sigma": { + "intercept": -0.62765582, + "raw_sigma_coef": 0.38492261, + "spread_coef": 0.29095921, + "peak_flag_coef": 0.33966327, + "max_so_far_gap_coef": 0.08357376 + } + }, + "sigma_constraints": { + "min_ratio": 0.85, + "max_ratio": 1.35, + "absolute_min": 0.25, + "absolute_max": 3.0 + }, + "selection_guardrails": { + "max_mae_increase": 0.02, + "max_bucket_hit_drop": 0.01, + "max_bucket_brier_increase": 0.05 + }, + "blending": { + "alpha_mu": 0.25, + "alpha_sigma": 1.0 + }, + "cities": { + "istanbul": { + "samples": 20, + "mu_bias": -0.438949, + "sigma_scale": 0.553793, + "confidence": 1.0 + }, + "buenos aires": { + "samples": 22, + "mu_bias": -1.044971, + "sigma_scale": 1.389753, + "confidence": 1.0 + }, + "ankara": { + "samples": 23, + "mu_bias": 0.067406, + "sigma_scale": 1.270298, + "confidence": 1.0 + }, + "london": { + "samples": 22, + "mu_bias": -0.010833, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "paris": { + "samples": 22, + "mu_bias": -0.21677, + "sigma_scale": 1.095669, + "confidence": 1.0 + }, + "new york": { + "samples": 21, + "mu_bias": 1.072189, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "lagos": { + "samples": 10, + "mu_bias": -0.092732, + "sigma_scale": 1.558646, + "confidence": 1.0 + }, + "madrid": { + "samples": 22, + "mu_bias": -0.627862, + "sigma_scale": 1.761239, + "confidence": 1.0 + }, + "toronto": { + "samples": 22, + "mu_bias": -0.424101, + "sigma_scale": 1.326675, + "confidence": 1.0 + }, + "wellington": { + "samples": 22, + "mu_bias": -0.347796, + "sigma_scale": 0.893459, + "confidence": 1.0 + }, + "los angeles": { + "samples": 20, + "mu_bias": -0.659983, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "san francisco": { + "samples": 20, + "mu_bias": 0.941818, + "sigma_scale": 1.979403, + "confidence": 1.0 + }, + "aurora": { + "samples": 20, + "mu_bias": 2.040009, + "sigma_scale": 0.982492, + "confidence": 1.0 + }, + "austin": { + "samples": 20, + "mu_bias": 0.990023, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "houston": { + "samples": 20, + "mu_bias": 0.188787, + "sigma_scale": 1.819594, + "confidence": 1.0 + }, + "chicago": { + "samples": 21, + "mu_bias": 2.285098, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "dallas": { + "samples": 21, + "mu_bias": 0.258095, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "atlanta": { + "samples": 22, + "mu_bias": -2.093105, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "seattle": { + "samples": 21, + "mu_bias": -0.982188, + "sigma_scale": 1.695172, + "confidence": 1.0 + }, + "sao paulo": { + "samples": 22, + "mu_bias": -0.786242, + "sigma_scale": 1.427037, + "confidence": 1.0 + }, + "munich": { + "samples": 22, + "mu_bias": 0.096559, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "milan": { + "samples": 23, + "mu_bias": -0.045038, + "sigma_scale": 1.626331, + "confidence": 1.0 + }, + "warsaw": { + "samples": 23, + "mu_bias": -0.50853, + "sigma_scale": 1.239146, + "confidence": 1.0 + }, + "helsinki": { + "samples": 17, + "mu_bias": 0.442247, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "amsterdam": { + "samples": 17, + "mu_bias": 0.129889, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "panama city": { + "samples": 17, + "mu_bias": -0.040792, + "sigma_scale": 1.393259, + "confidence": 1.0 + }, + "cape town": { + "samples": 10, + "mu_bias": 0.507293, + "sigma_scale": 1.420004, + "confidence": 1.0 + }, + "miami": { + "samples": 22, + "mu_bias": -2.736316, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "manila": { + "samples": 5, + "mu_bias": 1.772229, + "sigma_scale": 2.0, + "confidence": 0.625 + }, + "singapore": { + "samples": 22, + "mu_bias": 0.289639, + "sigma_scale": 1.124006, + "confidence": 1.0 + }, + "tokyo": { + "samples": 22, + "mu_bias": -0.332129, + "sigma_scale": 0.729483, + "confidence": 1.0 + }, + "seoul": { + "samples": 22, + "mu_bias": 1.6533, + "sigma_scale": 0.82648, + "confidence": 1.0 + }, + "busan": { + "samples": 16, + "mu_bias": 0.343239, + "sigma_scale": 1.886868, + "confidence": 1.0 + }, + "mexico city": { + "samples": 20, + "mu_bias": 0.087122, + "sigma_scale": 1.108884, + "confidence": 1.0 + }, + "shanghai": { + "samples": 23, + "mu_bias": 0.348022, + "sigma_scale": 0.810527, + "confidence": 1.0 + }, + "beijing": { + "samples": 20, + "mu_bias": -0.288176, + "sigma_scale": 0.952972, + "confidence": 1.0 + }, + "shenzhen": { + "samples": 21, + "mu_bias": -0.006114, + "sigma_scale": 0.728494, + "confidence": 1.0 + }, + "wuhan": { + "samples": 20, + "mu_bias": -0.771154, + "sigma_scale": 1.30329, + "confidence": 1.0 + }, + "chengdu": { + "samples": 20, + "mu_bias": 0.445906, + "sigma_scale": 0.796989, + "confidence": 1.0 + }, + "chongqing": { + "samples": 20, + "mu_bias": -0.021301, + "sigma_scale": 1.106566, + "confidence": 1.0 + }, + "hong kong": { + "samples": 24, + "mu_bias": 0.37527, + "sigma_scale": 0.517756, + "confidence": 1.0 + }, + "taipei": { + "samples": 23, + "mu_bias": 0.600671, + "sigma_scale": 0.5, + "confidence": 1.0 + }, + "guangzhou": { + "samples": 6, + "mu_bias": 0.7082, + "sigma_scale": 1.975103, + "confidence": 0.75 + }, + "lau fau shan": { + "samples": 19, + "mu_bias": 0.41371, + "sigma_scale": 0.656578, + "confidence": 1.0 + }, + "kuala lumpur": { + "samples": 16, + "mu_bias": 0.499197, + "sigma_scale": 1.361171, + "confidence": 1.0 + }, + "jakarta": { + "samples": 16, + "mu_bias": 0.293537, + "sigma_scale": 0.751498, + "confidence": 1.0 + }, + "lucknow": { + "samples": 22, + "mu_bias": -1.593437, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "karachi": { + "samples": 5, + "mu_bias": -0.035595, + "sigma_scale": 0.588612, + "confidence": 0.625 + }, + "jeddah": { + "samples": 10, + "mu_bias": 0.516879, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "moscow": { + "samples": 20, + "mu_bias": -0.609602, + "sigma_scale": 0.783723, + "confidence": 1.0 + }, + "tel aviv": { + "samples": 22, + "mu_bias": 0.436837, + "sigma_scale": 1.46524, + "confidence": 1.0 + } + }, + "metrics": { + "sample_count": 978, + "mean_crps": 1.330733, + "legacy_mean_crps": 1.376936, + "legacy_mean_mae": 1.589438, + "legacy_bucket_hit_rate": 0.43865, + "legacy_bucket_brier": 0.824571, + "selected_mean_crps": 1.311763, + "selected_mean_mae": 1.553658, + "selected_bucket_hit_rate": 0.432515, + "selected_bucket_brier": 0.789817, + "selected_score": 2.910718, + "legacy_score": 2.996333, + "filled_actual_from_history": 0, + "settlement_history_city_count": 30, + "legacy_archive_samples": 54 + }, + "source": "artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421121920\\default.json" +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/evaluation_report.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/evaluation_report.json new file mode 100644 index 00000000..8048c6be --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421121920/evaluation_report.json @@ -0,0 +1,482 @@ +{ + "summary": { + "sample_count": 978, + "filled_actual_from_history": 0, + "legacy": { + "mean_crps": 1.376936, + "mean_mae": 1.589438, + "bucket_hit_rate": 0.43865 + }, + "emos": { + "mean_crps": 1.319727, + "mean_mae": 1.554869, + "bucket_hit_rate": 0.43456 + }, + "delta": { + "crps": -0.05721, + "mae": -0.034568, + "bucket_hit_rate": -0.00409 + } + }, + "by_city": { + "amsterdam": { + "samples": 17, + "legacy_mean_crps": 0.868937, + "emos_mean_crps": 0.847633, + "legacy_mean_mae": 0.983529, + "emos_mean_mae": 0.979456, + "legacy_bucket_hit_rate": 0.529412, + "emos_bucket_hit_rate": 0.470588 + }, + "ankara": { + "samples": 23, + "legacy_mean_crps": 0.610138, + "emos_mean_crps": 0.569029, + "legacy_mean_mae": 0.726087, + "emos_mean_mae": 0.731312, + "legacy_bucket_hit_rate": 0.434783, + "emos_bucket_hit_rate": 0.391304 + }, + "atlanta": { + "samples": 22, + "legacy_mean_crps": 4.304444, + "emos_mean_crps": 4.435438, + "legacy_mean_mae": 4.79, + "emos_mean_mae": 4.992948, + "legacy_bucket_hit_rate": 0.181818, + "emos_bucket_hit_rate": 0.090909 + }, + "aurora": { + "samples": 20, + "legacy_mean_crps": 3.989577, + "emos_mean_crps": 3.688746, + "legacy_mean_mae": 4.5115, + "emos_mean_mae": 4.454993, + "legacy_bucket_hit_rate": 0.1, + "emos_bucket_hit_rate": 0.0 + }, + "austin": { + "samples": 20, + "legacy_mean_crps": 2.261424, + "emos_mean_crps": 2.109386, + "legacy_mean_mae": 2.422, + "emos_mean_mae": 2.377675, + "legacy_bucket_hit_rate": 0.45, + "emos_bucket_hit_rate": 0.4 + }, + "beijing": { + "samples": 20, + "legacy_mean_crps": 1.116518, + "emos_mean_crps": 1.123144, + "legacy_mean_mae": 1.4955, + "emos_mean_mae": 1.468371, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.4 + }, + "buenos aires": { + "samples": 22, + "legacy_mean_crps": 1.632274, + "emos_mean_crps": 1.657172, + "legacy_mean_mae": 1.885, + "emos_mean_mae": 1.875542, + "legacy_bucket_hit_rate": 0.454545, + "emos_bucket_hit_rate": 0.5 + }, + "busan": { + "samples": 16, + "legacy_mean_crps": 1.066146, + "emos_mean_crps": 0.99263, + "legacy_mean_mae": 1.304375, + "emos_mean_mae": 1.259453, + "legacy_bucket_hit_rate": 0.375, + "emos_bucket_hit_rate": 0.375 + }, + "cape town": { + "samples": 10, + "legacy_mean_crps": 0.7369, + "emos_mean_crps": 0.646728, + "legacy_mean_mae": 0.882, + "emos_mean_mae": 0.809407, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.4 + }, + "chengdu": { + "samples": 20, + "legacy_mean_crps": 1.009046, + "emos_mean_crps": 0.964076, + "legacy_mean_mae": 1.1835, + "emos_mean_mae": 1.155294, + "legacy_bucket_hit_rate": 0.5, + "emos_bucket_hit_rate": 0.5 + }, + "chicago": { + "samples": 21, + "legacy_mean_crps": 3.052956, + "emos_mean_crps": 2.842851, + "legacy_mean_mae": 3.398095, + "emos_mean_mae": 3.312975, + "legacy_bucket_hit_rate": 0.285714, + "emos_bucket_hit_rate": 0.095238 + }, + "chongqing": { + "samples": 20, + "legacy_mean_crps": 1.239078, + "emos_mean_crps": 1.231804, + "legacy_mean_mae": 1.4955, + "emos_mean_mae": 1.501486, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.4 + }, + "dallas": { + "samples": 21, + "legacy_mean_crps": 1.795116, + "emos_mean_crps": 1.700865, + "legacy_mean_mae": 1.979048, + "emos_mean_mae": 2.017151, + "legacy_bucket_hit_rate": 0.285714, + "emos_bucket_hit_rate": 0.238095 + }, + "guangzhou": { + "samples": 6, + "legacy_mean_crps": 1.586654, + "emos_mean_crps": 1.565936, + "legacy_mean_mae": 1.991667, + "emos_mean_mae": 1.995792, + "legacy_bucket_hit_rate": 0.166667, + "emos_bucket_hit_rate": 0.0 + }, + "helsinki": { + "samples": 17, + "legacy_mean_crps": 0.626777, + "emos_mean_crps": 0.489382, + "legacy_mean_mae": 0.785294, + "emos_mean_mae": 0.640606, + "legacy_bucket_hit_rate": 0.470588, + "emos_bucket_hit_rate": 0.529412 + }, + "hong kong": { + "samples": 24, + "legacy_mean_crps": 0.531004, + "emos_mean_crps": 0.473302, + "legacy_mean_mae": 0.552917, + "emos_mean_mae": 0.52881, + "legacy_bucket_hit_rate": 0.666667, + "emos_bucket_hit_rate": 0.75 + }, + "houston": { + "samples": 20, + "legacy_mean_crps": 1.488859, + "emos_mean_crps": 1.412564, + "legacy_mean_mae": 1.8085, + "emos_mean_mae": 1.80359, + "legacy_bucket_hit_rate": 0.25, + "emos_bucket_hit_rate": 0.2 + }, + "istanbul": { + "samples": 20, + "legacy_mean_crps": 0.586067, + "emos_mean_crps": 0.546494, + "legacy_mean_mae": 0.6225, + "emos_mean_mae": 0.557255, + "legacy_bucket_hit_rate": 0.7, + "emos_bucket_hit_rate": 0.7 + }, + "jakarta": { + "samples": 16, + "legacy_mean_crps": 0.620583, + "emos_mean_crps": 0.553084, + "legacy_mean_mae": 0.875625, + "emos_mean_mae": 0.793719, + "legacy_bucket_hit_rate": 0.4375, + "emos_bucket_hit_rate": 0.4375 + }, + "jeddah": { + "samples": 10, + "legacy_mean_crps": 1.822658, + "emos_mean_crps": 1.674198, + "legacy_mean_mae": 2.277, + "emos_mean_mae": 2.192556, + "legacy_bucket_hit_rate": 0.2, + "emos_bucket_hit_rate": 0.1 + }, + "karachi": { + "samples": 5, + "legacy_mean_crps": 0.361328, + "emos_mean_crps": 0.326866, + "legacy_mean_mae": 0.398, + "emos_mean_mae": 0.336276, + "legacy_bucket_hit_rate": 0.8, + "emos_bucket_hit_rate": 0.8 + }, + "kuala lumpur": { + "samples": 16, + "legacy_mean_crps": 0.927888, + "emos_mean_crps": 0.86633, + "legacy_mean_mae": 1.146875, + "emos_mean_mae": 1.083193, + "legacy_bucket_hit_rate": 0.375, + "emos_bucket_hit_rate": 0.5 + }, + "lagos": { + "samples": 10, + "legacy_mean_crps": 1.174893, + "emos_mean_crps": 1.204827, + "legacy_mean_mae": 1.592, + "emos_mean_mae": 1.586221, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "lau fau shan": { + "samples": 19, + "legacy_mean_crps": 0.713018, + "emos_mean_crps": 0.63244, + "legacy_mean_mae": 0.706316, + "emos_mean_mae": 0.641667, + "legacy_bucket_hit_rate": 0.578947, + "emos_bucket_hit_rate": 0.578947 + }, + "london": { + "samples": 22, + "legacy_mean_crps": 1.298133, + "emos_mean_crps": 1.267504, + "legacy_mean_mae": 1.429091, + "emos_mean_mae": 1.422989, + "legacy_bucket_hit_rate": 0.409091, + "emos_bucket_hit_rate": 0.409091 + }, + "los angeles": { + "samples": 20, + "legacy_mean_crps": 2.051516, + "emos_mean_crps": 2.006381, + "legacy_mean_mae": 2.4145, + "emos_mean_mae": 2.366065, + "legacy_bucket_hit_rate": 0.3, + "emos_bucket_hit_rate": 0.3 + }, + "lucknow": { + "samples": 22, + "legacy_mean_crps": 1.134917, + "emos_mean_crps": 0.890494, + "legacy_mean_mae": 1.532727, + "emos_mean_mae": 1.201547, + "legacy_bucket_hit_rate": 0.136364, + "emos_bucket_hit_rate": 0.272727 + }, + "madrid": { + "samples": 22, + "legacy_mean_crps": 1.196523, + "emos_mean_crps": 1.197589, + "legacy_mean_mae": 1.380909, + "emos_mean_mae": 1.383717, + "legacy_bucket_hit_rate": 0.545455, + "emos_bucket_hit_rate": 0.636364 + }, + "manila": { + "samples": 5, + "legacy_mean_crps": 1.9342, + "emos_mean_crps": 1.510621, + "legacy_mean_mae": 2.408, + "emos_mean_mae": 1.914957, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "mexico city": { + "samples": 20, + "legacy_mean_crps": 0.560208, + "emos_mean_crps": 0.506071, + "legacy_mean_mae": 0.713, + "emos_mean_mae": 0.653865, + "legacy_bucket_hit_rate": 0.55, + "emos_bucket_hit_rate": 0.5 + }, + "miami": { + "samples": 22, + "legacy_mean_crps": 3.359106, + "emos_mean_crps": 3.561458, + "legacy_mean_mae": 3.660455, + "emos_mean_mae": 3.936442, + "legacy_bucket_hit_rate": 0.454545, + "emos_bucket_hit_rate": 0.227273 + }, + "milan": { + "samples": 23, + "legacy_mean_crps": 1.184521, + "emos_mean_crps": 1.055058, + "legacy_mean_mae": 1.284348, + "emos_mean_mae": 1.220263, + "legacy_bucket_hit_rate": 0.304348, + "emos_bucket_hit_rate": 0.434783 + }, + "moscow": { + "samples": 20, + "legacy_mean_crps": 0.617727, + "emos_mean_crps": 0.578817, + "legacy_mean_mae": 0.6375, + "emos_mean_mae": 0.576256, + "legacy_bucket_hit_rate": 0.75, + "emos_bucket_hit_rate": 0.75 + }, + "munich": { + "samples": 22, + "legacy_mean_crps": 1.101444, + "emos_mean_crps": 1.049123, + "legacy_mean_mae": 1.298182, + "emos_mean_mae": 1.285139, + "legacy_bucket_hit_rate": 0.227273, + "emos_bucket_hit_rate": 0.227273 + }, + "new york": { + "samples": 21, + "legacy_mean_crps": 3.428596, + "emos_mean_crps": 3.466282, + "legacy_mean_mae": 3.988571, + "emos_mean_mae": 4.038895, + "legacy_bucket_hit_rate": 0.238095, + "emos_bucket_hit_rate": 0.238095 + }, + "panama city": { + "samples": 17, + "legacy_mean_crps": 0.347679, + "emos_mean_crps": 0.287754, + "legacy_mean_mae": 0.476471, + "emos_mean_mae": 0.391782, + "legacy_bucket_hit_rate": 0.588235, + "emos_bucket_hit_rate": 0.823529 + }, + "paris": { + "samples": 22, + "legacy_mean_crps": 0.718617, + "emos_mean_crps": 0.691394, + "legacy_mean_mae": 0.680455, + "emos_mean_mae": 0.696494, + "legacy_bucket_hit_rate": 0.772727, + "emos_bucket_hit_rate": 0.772727 + }, + "san francisco": { + "samples": 20, + "legacy_mean_crps": 2.623636, + "emos_mean_crps": 2.475182, + "legacy_mean_mae": 2.9075, + "emos_mean_mae": 2.837902, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.25 + }, + "sao paulo": { + "samples": 22, + "legacy_mean_crps": 0.930626, + "emos_mean_crps": 0.931492, + "legacy_mean_mae": 1.16, + "emos_mean_mae": 1.123038, + "legacy_bucket_hit_rate": 0.590909, + "emos_bucket_hit_rate": 0.5 + }, + "seattle": { + "samples": 21, + "legacy_mean_crps": 1.378395, + "emos_mean_crps": 1.333721, + "legacy_mean_mae": 1.601905, + "emos_mean_mae": 1.565253, + "legacy_bucket_hit_rate": 0.428571, + "emos_bucket_hit_rate": 0.380952 + }, + "seoul": { + "samples": 22, + "legacy_mean_crps": 1.449956, + "emos_mean_crps": 1.264612, + "legacy_mean_mae": 1.817727, + "emos_mean_mae": 1.604882, + "legacy_bucket_hit_rate": 0.409091, + "emos_bucket_hit_rate": 0.454545 + }, + "shanghai": { + "samples": 23, + "legacy_mean_crps": 1.237937, + "emos_mean_crps": 1.196583, + "legacy_mean_mae": 1.458261, + "emos_mean_mae": 1.430445, + "legacy_bucket_hit_rate": 0.565217, + "emos_bucket_hit_rate": 0.608696 + }, + "shenzhen": { + "samples": 21, + "legacy_mean_crps": 0.481109, + "emos_mean_crps": 0.444093, + "legacy_mean_mae": 0.608095, + "emos_mean_mae": 0.581253, + "legacy_bucket_hit_rate": 0.619048, + "emos_bucket_hit_rate": 0.666667 + }, + "singapore": { + "samples": 22, + "legacy_mean_crps": 0.852942, + "emos_mean_crps": 0.758454, + "legacy_mean_mae": 1.020909, + "emos_mean_mae": 0.915982, + "legacy_bucket_hit_rate": 0.409091, + "emos_bucket_hit_rate": 0.454545 + }, + "taipei": { + "samples": 23, + "legacy_mean_crps": 0.801052, + "emos_mean_crps": 0.721468, + "legacy_mean_mae": 0.85087, + "emos_mean_mae": 0.814252, + "legacy_bucket_hit_rate": 0.478261, + "emos_bucket_hit_rate": 0.391304 + }, + "tel aviv": { + "samples": 22, + "legacy_mean_crps": 0.770763, + "emos_mean_crps": 0.672956, + "legacy_mean_mae": 0.904091, + "emos_mean_mae": 0.843671, + "legacy_bucket_hit_rate": 0.454545, + "emos_bucket_hit_rate": 0.5 + }, + "tokyo": { + "samples": 22, + "legacy_mean_crps": 0.651385, + "emos_mean_crps": 0.641684, + "legacy_mean_mae": 0.798182, + "emos_mean_mae": 0.787542, + "legacy_bucket_hit_rate": 0.545455, + "emos_bucket_hit_rate": 0.590909 + }, + "toronto": { + "samples": 22, + "legacy_mean_crps": 2.733044, + "emos_mean_crps": 2.714624, + "legacy_mean_mae": 2.981818, + "emos_mean_mae": 2.969098, + "legacy_bucket_hit_rate": 0.318182, + "emos_bucket_hit_rate": 0.409091 + }, + "warsaw": { + "samples": 23, + "legacy_mean_crps": 0.668776, + "emos_mean_crps": 0.665164, + "legacy_mean_mae": 0.796087, + "emos_mean_mae": 0.796958, + "legacy_bucket_hit_rate": 0.652174, + "emos_bucket_hit_rate": 0.608696 + }, + "wellington": { + "samples": 22, + "legacy_mean_crps": 0.731346, + "emos_mean_crps": 0.726977, + "legacy_mean_mae": 0.895, + "emos_mean_mae": 0.892558, + "legacy_bucket_hit_rate": 0.5, + "emos_bucket_hit_rate": 0.5 + }, + "wuhan": { + "samples": 20, + "legacy_mean_crps": 0.925443, + "emos_mean_crps": 0.897806, + "legacy_mean_mae": 1.0375, + "emos_mean_mae": 1.003133, + "legacy_bucket_hit_rate": 0.65, + "emos_bucket_hit_rate": 0.65 + } + } +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/decision_report.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/decision_report.json new file mode 100644 index 00000000..a90686bc --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/decision_report.json @@ -0,0 +1,62 @@ +{ + "ok": true, + "version": "emos-auto-20260421122743", + "generated_at": "2026-04-21T12:28:08.021388+00:00", + "candidate_dir": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743", + "candidate_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json", + "evaluation_path": "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\evaluation_report.json", + "target_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\default.json", + "promote_requested": true, + "promoted": true, + "backup_path": "E:\\web\\PolyWeather\\artifacts\\probability_calibration\\default.backup-20260421122808.json", + "decision": { + "decision": "promote", + "ready_for_promotion": true, + "blocking_reasons": [], + "thresholds": { + "min_samples": 50, + "max_delta_crps": 0.0, + "max_delta_mae": 0.05, + "min_delta_bucket_hit_rate": -0.05 + }, + "metrics": { + "sample_count": 978, + "delta_crps": -0.05721, + "delta_mae": -0.034568, + "delta_bucket_hit_rate": -0.00409 + } + }, + "fit": { + "command": [ + "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + "scripts/fit_probability_calibration.py", + "--output", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json", + "--version", + "emos-auto-20260421122743", + "--verbose", + "--snapshot-limit", + "50000" + ], + "returncode": 0, + "stdout": "", + "stderr": "" + }, + "evaluate": { + "command": [ + "C:\\Users\\Yang\\AppData\\Local\\Programs\\Python\\Python38\\python.exe", + "scripts/evaluate_probability_calibration.py", + "--calibration-file", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json", + "--output", + "E:\\web\\PolyWeather\\artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\evaluation_report.json", + "--verbose", + "--snapshot-limit", + "50000" + ], + "returncode": 0, + "stdout": "", + "stderr": "" + }, + "tests": null +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/default.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/default.json new file mode 100644 index 00000000..ad104d58 --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/default.json @@ -0,0 +1,361 @@ +{ + "version": "emos-auto-20260421122743", + "trained_at": "2026-04-21T12:28:05.031165+00:00", + "global": { + "mu": { + "intercept": 0.34330438, + "raw_mu_coef": 0.48749629, + "deb_coef": 0.45733479, + "ens_median_coef": 0.05955297, + "max_so_far_gap_coef": -0.104935 + }, + "sigma": { + "intercept": -0.62765582, + "raw_sigma_coef": 0.38492261, + "spread_coef": 0.29095921, + "peak_flag_coef": 0.33966327, + "max_so_far_gap_coef": 0.08357376 + } + }, + "sigma_constraints": { + "min_ratio": 0.85, + "max_ratio": 1.35, + "absolute_min": 0.25, + "absolute_max": 3.0 + }, + "selection_guardrails": { + "max_mae_increase": 0.02, + "max_bucket_hit_drop": 0.01, + "max_bucket_brier_increase": 0.05 + }, + "blending": { + "alpha_mu": 0.25, + "alpha_sigma": 1.0 + }, + "cities": { + "istanbul": { + "samples": 20, + "mu_bias": -0.438949, + "sigma_scale": 0.553793, + "confidence": 1.0 + }, + "buenos aires": { + "samples": 22, + "mu_bias": -1.044971, + "sigma_scale": 1.389753, + "confidence": 1.0 + }, + "ankara": { + "samples": 23, + "mu_bias": 0.067406, + "sigma_scale": 1.270298, + "confidence": 1.0 + }, + "london": { + "samples": 22, + "mu_bias": -0.010833, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "paris": { + "samples": 22, + "mu_bias": -0.21677, + "sigma_scale": 1.095669, + "confidence": 1.0 + }, + "new york": { + "samples": 21, + "mu_bias": 1.072189, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "lagos": { + "samples": 10, + "mu_bias": -0.092732, + "sigma_scale": 1.558646, + "confidence": 1.0 + }, + "madrid": { + "samples": 22, + "mu_bias": -0.627862, + "sigma_scale": 1.761239, + "confidence": 1.0 + }, + "toronto": { + "samples": 22, + "mu_bias": -0.424101, + "sigma_scale": 1.326675, + "confidence": 1.0 + }, + "wellington": { + "samples": 22, + "mu_bias": -0.347796, + "sigma_scale": 0.893459, + "confidence": 1.0 + }, + "los angeles": { + "samples": 20, + "mu_bias": -0.659983, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "san francisco": { + "samples": 20, + "mu_bias": 0.941818, + "sigma_scale": 1.979403, + "confidence": 1.0 + }, + "aurora": { + "samples": 20, + "mu_bias": 2.040009, + "sigma_scale": 0.982492, + "confidence": 1.0 + }, + "austin": { + "samples": 20, + "mu_bias": 0.990023, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "houston": { + "samples": 20, + "mu_bias": 0.188787, + "sigma_scale": 1.819594, + "confidence": 1.0 + }, + "chicago": { + "samples": 21, + "mu_bias": 2.285098, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "dallas": { + "samples": 21, + "mu_bias": 0.258095, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "atlanta": { + "samples": 22, + "mu_bias": -2.093105, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "seattle": { + "samples": 21, + "mu_bias": -0.982188, + "sigma_scale": 1.695172, + "confidence": 1.0 + }, + "sao paulo": { + "samples": 22, + "mu_bias": -0.786242, + "sigma_scale": 1.427037, + "confidence": 1.0 + }, + "munich": { + "samples": 22, + "mu_bias": 0.096559, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "milan": { + "samples": 23, + "mu_bias": -0.045038, + "sigma_scale": 1.626331, + "confidence": 1.0 + }, + "warsaw": { + "samples": 23, + "mu_bias": -0.50853, + "sigma_scale": 1.239146, + "confidence": 1.0 + }, + "helsinki": { + "samples": 17, + "mu_bias": 0.442247, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "amsterdam": { + "samples": 17, + "mu_bias": 0.129889, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "panama city": { + "samples": 17, + "mu_bias": -0.040792, + "sigma_scale": 1.393259, + "confidence": 1.0 + }, + "cape town": { + "samples": 10, + "mu_bias": 0.507293, + "sigma_scale": 1.420004, + "confidence": 1.0 + }, + "miami": { + "samples": 22, + "mu_bias": -2.736316, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "manila": { + "samples": 5, + "mu_bias": 1.772229, + "sigma_scale": 2.0, + "confidence": 0.625 + }, + "singapore": { + "samples": 22, + "mu_bias": 0.289639, + "sigma_scale": 1.124006, + "confidence": 1.0 + }, + "tokyo": { + "samples": 22, + "mu_bias": -0.332129, + "sigma_scale": 0.729483, + "confidence": 1.0 + }, + "seoul": { + "samples": 22, + "mu_bias": 1.6533, + "sigma_scale": 0.82648, + "confidence": 1.0 + }, + "busan": { + "samples": 16, + "mu_bias": 0.343239, + "sigma_scale": 1.886868, + "confidence": 1.0 + }, + "mexico city": { + "samples": 20, + "mu_bias": 0.087122, + "sigma_scale": 1.108884, + "confidence": 1.0 + }, + "shanghai": { + "samples": 23, + "mu_bias": 0.348022, + "sigma_scale": 0.810527, + "confidence": 1.0 + }, + "beijing": { + "samples": 20, + "mu_bias": -0.288176, + "sigma_scale": 0.952972, + "confidence": 1.0 + }, + "shenzhen": { + "samples": 21, + "mu_bias": -0.006114, + "sigma_scale": 0.728494, + "confidence": 1.0 + }, + "wuhan": { + "samples": 20, + "mu_bias": -0.771154, + "sigma_scale": 1.30329, + "confidence": 1.0 + }, + "chengdu": { + "samples": 20, + "mu_bias": 0.445906, + "sigma_scale": 0.796989, + "confidence": 1.0 + }, + "chongqing": { + "samples": 20, + "mu_bias": -0.021301, + "sigma_scale": 1.106566, + "confidence": 1.0 + }, + "hong kong": { + "samples": 24, + "mu_bias": 0.37527, + "sigma_scale": 0.517756, + "confidence": 1.0 + }, + "taipei": { + "samples": 23, + "mu_bias": 0.600671, + "sigma_scale": 0.5, + "confidence": 1.0 + }, + "guangzhou": { + "samples": 6, + "mu_bias": 0.7082, + "sigma_scale": 1.975103, + "confidence": 0.75 + }, + "lau fau shan": { + "samples": 19, + "mu_bias": 0.41371, + "sigma_scale": 0.656578, + "confidence": 1.0 + }, + "kuala lumpur": { + "samples": 16, + "mu_bias": 0.499197, + "sigma_scale": 1.361171, + "confidence": 1.0 + }, + "jakarta": { + "samples": 16, + "mu_bias": 0.293537, + "sigma_scale": 0.751498, + "confidence": 1.0 + }, + "lucknow": { + "samples": 22, + "mu_bias": -1.593437, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "karachi": { + "samples": 5, + "mu_bias": -0.035595, + "sigma_scale": 0.588612, + "confidence": 0.625 + }, + "jeddah": { + "samples": 10, + "mu_bias": 0.516879, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "moscow": { + "samples": 20, + "mu_bias": -0.609602, + "sigma_scale": 0.783723, + "confidence": 1.0 + }, + "tel aviv": { + "samples": 22, + "mu_bias": 0.436837, + "sigma_scale": 1.46524, + "confidence": 1.0 + } + }, + "metrics": { + "sample_count": 978, + "mean_crps": 1.330733, + "legacy_mean_crps": 1.376936, + "legacy_mean_mae": 1.589438, + "legacy_bucket_hit_rate": 0.43865, + "legacy_bucket_brier": 0.824571, + "selected_mean_crps": 1.311763, + "selected_mean_mae": 1.553658, + "selected_bucket_hit_rate": 0.432515, + "selected_bucket_brier": 0.789817, + "selected_score": 2.910718, + "legacy_score": 2.996333, + "filled_actual_from_history": 0, + "settlement_history_city_count": 30, + "legacy_archive_samples": 54 + }, + "source": "artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json" +} \ No newline at end of file diff --git a/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/evaluation_report.json b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/evaluation_report.json new file mode 100644 index 00000000..8048c6be --- /dev/null +++ b/artifacts/local_runtime/probability_calibration/candidates/emos-auto-20260421122743/evaluation_report.json @@ -0,0 +1,482 @@ +{ + "summary": { + "sample_count": 978, + "filled_actual_from_history": 0, + "legacy": { + "mean_crps": 1.376936, + "mean_mae": 1.589438, + "bucket_hit_rate": 0.43865 + }, + "emos": { + "mean_crps": 1.319727, + "mean_mae": 1.554869, + "bucket_hit_rate": 0.43456 + }, + "delta": { + "crps": -0.05721, + "mae": -0.034568, + "bucket_hit_rate": -0.00409 + } + }, + "by_city": { + "amsterdam": { + "samples": 17, + "legacy_mean_crps": 0.868937, + "emos_mean_crps": 0.847633, + "legacy_mean_mae": 0.983529, + "emos_mean_mae": 0.979456, + "legacy_bucket_hit_rate": 0.529412, + "emos_bucket_hit_rate": 0.470588 + }, + "ankara": { + "samples": 23, + "legacy_mean_crps": 0.610138, + "emos_mean_crps": 0.569029, + "legacy_mean_mae": 0.726087, + "emos_mean_mae": 0.731312, + "legacy_bucket_hit_rate": 0.434783, + "emos_bucket_hit_rate": 0.391304 + }, + "atlanta": { + "samples": 22, + "legacy_mean_crps": 4.304444, + "emos_mean_crps": 4.435438, + "legacy_mean_mae": 4.79, + "emos_mean_mae": 4.992948, + "legacy_bucket_hit_rate": 0.181818, + "emos_bucket_hit_rate": 0.090909 + }, + "aurora": { + "samples": 20, + "legacy_mean_crps": 3.989577, + "emos_mean_crps": 3.688746, + "legacy_mean_mae": 4.5115, + "emos_mean_mae": 4.454993, + "legacy_bucket_hit_rate": 0.1, + "emos_bucket_hit_rate": 0.0 + }, + "austin": { + "samples": 20, + "legacy_mean_crps": 2.261424, + "emos_mean_crps": 2.109386, + "legacy_mean_mae": 2.422, + "emos_mean_mae": 2.377675, + "legacy_bucket_hit_rate": 0.45, + "emos_bucket_hit_rate": 0.4 + }, + "beijing": { + "samples": 20, + "legacy_mean_crps": 1.116518, + "emos_mean_crps": 1.123144, + "legacy_mean_mae": 1.4955, + "emos_mean_mae": 1.468371, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.4 + }, + "buenos aires": { + "samples": 22, + "legacy_mean_crps": 1.632274, + "emos_mean_crps": 1.657172, + "legacy_mean_mae": 1.885, + "emos_mean_mae": 1.875542, + "legacy_bucket_hit_rate": 0.454545, + "emos_bucket_hit_rate": 0.5 + }, + "busan": { + "samples": 16, + "legacy_mean_crps": 1.066146, + "emos_mean_crps": 0.99263, + "legacy_mean_mae": 1.304375, + "emos_mean_mae": 1.259453, + "legacy_bucket_hit_rate": 0.375, + "emos_bucket_hit_rate": 0.375 + }, + "cape town": { + "samples": 10, + "legacy_mean_crps": 0.7369, + "emos_mean_crps": 0.646728, + "legacy_mean_mae": 0.882, + "emos_mean_mae": 0.809407, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.4 + }, + "chengdu": { + "samples": 20, + "legacy_mean_crps": 1.009046, + "emos_mean_crps": 0.964076, + "legacy_mean_mae": 1.1835, + "emos_mean_mae": 1.155294, + "legacy_bucket_hit_rate": 0.5, + "emos_bucket_hit_rate": 0.5 + }, + "chicago": { + "samples": 21, + "legacy_mean_crps": 3.052956, + "emos_mean_crps": 2.842851, + "legacy_mean_mae": 3.398095, + "emos_mean_mae": 3.312975, + "legacy_bucket_hit_rate": 0.285714, + "emos_bucket_hit_rate": 0.095238 + }, + "chongqing": { + "samples": 20, + "legacy_mean_crps": 1.239078, + "emos_mean_crps": 1.231804, + "legacy_mean_mae": 1.4955, + "emos_mean_mae": 1.501486, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.4 + }, + "dallas": { + "samples": 21, + "legacy_mean_crps": 1.795116, + "emos_mean_crps": 1.700865, + "legacy_mean_mae": 1.979048, + "emos_mean_mae": 2.017151, + "legacy_bucket_hit_rate": 0.285714, + "emos_bucket_hit_rate": 0.238095 + }, + "guangzhou": { + "samples": 6, + "legacy_mean_crps": 1.586654, + "emos_mean_crps": 1.565936, + "legacy_mean_mae": 1.991667, + "emos_mean_mae": 1.995792, + "legacy_bucket_hit_rate": 0.166667, + "emos_bucket_hit_rate": 0.0 + }, + "helsinki": { + "samples": 17, + "legacy_mean_crps": 0.626777, + "emos_mean_crps": 0.489382, + "legacy_mean_mae": 0.785294, + "emos_mean_mae": 0.640606, + "legacy_bucket_hit_rate": 0.470588, + "emos_bucket_hit_rate": 0.529412 + }, + "hong kong": { + "samples": 24, + "legacy_mean_crps": 0.531004, + "emos_mean_crps": 0.473302, + "legacy_mean_mae": 0.552917, + "emos_mean_mae": 0.52881, + "legacy_bucket_hit_rate": 0.666667, + "emos_bucket_hit_rate": 0.75 + }, + "houston": { + "samples": 20, + "legacy_mean_crps": 1.488859, + "emos_mean_crps": 1.412564, + "legacy_mean_mae": 1.8085, + "emos_mean_mae": 1.80359, + "legacy_bucket_hit_rate": 0.25, + "emos_bucket_hit_rate": 0.2 + }, + "istanbul": { + "samples": 20, + "legacy_mean_crps": 0.586067, + "emos_mean_crps": 0.546494, + "legacy_mean_mae": 0.6225, + "emos_mean_mae": 0.557255, + "legacy_bucket_hit_rate": 0.7, + "emos_bucket_hit_rate": 0.7 + }, + "jakarta": { + "samples": 16, + "legacy_mean_crps": 0.620583, + "emos_mean_crps": 0.553084, + "legacy_mean_mae": 0.875625, + "emos_mean_mae": 0.793719, + "legacy_bucket_hit_rate": 0.4375, + "emos_bucket_hit_rate": 0.4375 + }, + "jeddah": { + "samples": 10, + "legacy_mean_crps": 1.822658, + "emos_mean_crps": 1.674198, + "legacy_mean_mae": 2.277, + "emos_mean_mae": 2.192556, + "legacy_bucket_hit_rate": 0.2, + "emos_bucket_hit_rate": 0.1 + }, + "karachi": { + "samples": 5, + "legacy_mean_crps": 0.361328, + "emos_mean_crps": 0.326866, + "legacy_mean_mae": 0.398, + "emos_mean_mae": 0.336276, + "legacy_bucket_hit_rate": 0.8, + "emos_bucket_hit_rate": 0.8 + }, + "kuala lumpur": { + "samples": 16, + "legacy_mean_crps": 0.927888, + "emos_mean_crps": 0.86633, + "legacy_mean_mae": 1.146875, + "emos_mean_mae": 1.083193, + "legacy_bucket_hit_rate": 0.375, + "emos_bucket_hit_rate": 0.5 + }, + "lagos": { + "samples": 10, + "legacy_mean_crps": 1.174893, + "emos_mean_crps": 1.204827, + "legacy_mean_mae": 1.592, + "emos_mean_mae": 1.586221, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "lau fau shan": { + "samples": 19, + "legacy_mean_crps": 0.713018, + "emos_mean_crps": 0.63244, + "legacy_mean_mae": 0.706316, + "emos_mean_mae": 0.641667, + "legacy_bucket_hit_rate": 0.578947, + "emos_bucket_hit_rate": 0.578947 + }, + "london": { + "samples": 22, + "legacy_mean_crps": 1.298133, + "emos_mean_crps": 1.267504, + "legacy_mean_mae": 1.429091, + "emos_mean_mae": 1.422989, + "legacy_bucket_hit_rate": 0.409091, + "emos_bucket_hit_rate": 0.409091 + }, + "los angeles": { + "samples": 20, + "legacy_mean_crps": 2.051516, + "emos_mean_crps": 2.006381, + "legacy_mean_mae": 2.4145, + "emos_mean_mae": 2.366065, + "legacy_bucket_hit_rate": 0.3, + "emos_bucket_hit_rate": 0.3 + }, + "lucknow": { + "samples": 22, + "legacy_mean_crps": 1.134917, + "emos_mean_crps": 0.890494, + "legacy_mean_mae": 1.532727, + "emos_mean_mae": 1.201547, + "legacy_bucket_hit_rate": 0.136364, + "emos_bucket_hit_rate": 0.272727 + }, + "madrid": { + "samples": 22, + "legacy_mean_crps": 1.196523, + "emos_mean_crps": 1.197589, + "legacy_mean_mae": 1.380909, + "emos_mean_mae": 1.383717, + "legacy_bucket_hit_rate": 0.545455, + "emos_bucket_hit_rate": 0.636364 + }, + "manila": { + "samples": 5, + "legacy_mean_crps": 1.9342, + "emos_mean_crps": 1.510621, + "legacy_mean_mae": 2.408, + "emos_mean_mae": 1.914957, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "mexico city": { + "samples": 20, + "legacy_mean_crps": 0.560208, + "emos_mean_crps": 0.506071, + "legacy_mean_mae": 0.713, + "emos_mean_mae": 0.653865, + "legacy_bucket_hit_rate": 0.55, + "emos_bucket_hit_rate": 0.5 + }, + "miami": { + "samples": 22, + "legacy_mean_crps": 3.359106, + "emos_mean_crps": 3.561458, + "legacy_mean_mae": 3.660455, + "emos_mean_mae": 3.936442, + "legacy_bucket_hit_rate": 0.454545, + "emos_bucket_hit_rate": 0.227273 + }, + "milan": { + "samples": 23, + "legacy_mean_crps": 1.184521, + "emos_mean_crps": 1.055058, + "legacy_mean_mae": 1.284348, + "emos_mean_mae": 1.220263, + "legacy_bucket_hit_rate": 0.304348, + "emos_bucket_hit_rate": 0.434783 + }, + "moscow": { + "samples": 20, + "legacy_mean_crps": 0.617727, + "emos_mean_crps": 0.578817, + "legacy_mean_mae": 0.6375, + "emos_mean_mae": 0.576256, + "legacy_bucket_hit_rate": 0.75, + "emos_bucket_hit_rate": 0.75 + }, + "munich": { + "samples": 22, + "legacy_mean_crps": 1.101444, + "emos_mean_crps": 1.049123, + "legacy_mean_mae": 1.298182, + "emos_mean_mae": 1.285139, + "legacy_bucket_hit_rate": 0.227273, + "emos_bucket_hit_rate": 0.227273 + }, + "new york": { + "samples": 21, + "legacy_mean_crps": 3.428596, + "emos_mean_crps": 3.466282, + "legacy_mean_mae": 3.988571, + "emos_mean_mae": 4.038895, + "legacy_bucket_hit_rate": 0.238095, + "emos_bucket_hit_rate": 0.238095 + }, + "panama city": { + "samples": 17, + "legacy_mean_crps": 0.347679, + "emos_mean_crps": 0.287754, + "legacy_mean_mae": 0.476471, + "emos_mean_mae": 0.391782, + "legacy_bucket_hit_rate": 0.588235, + "emos_bucket_hit_rate": 0.823529 + }, + "paris": { + "samples": 22, + "legacy_mean_crps": 0.718617, + "emos_mean_crps": 0.691394, + "legacy_mean_mae": 0.680455, + "emos_mean_mae": 0.696494, + "legacy_bucket_hit_rate": 0.772727, + "emos_bucket_hit_rate": 0.772727 + }, + "san francisco": { + "samples": 20, + "legacy_mean_crps": 2.623636, + "emos_mean_crps": 2.475182, + "legacy_mean_mae": 2.9075, + "emos_mean_mae": 2.837902, + "legacy_bucket_hit_rate": 0.4, + "emos_bucket_hit_rate": 0.25 + }, + "sao paulo": { + "samples": 22, + "legacy_mean_crps": 0.930626, + "emos_mean_crps": 0.931492, + "legacy_mean_mae": 1.16, + "emos_mean_mae": 1.123038, + "legacy_bucket_hit_rate": 0.590909, + "emos_bucket_hit_rate": 0.5 + }, + "seattle": { + "samples": 21, + "legacy_mean_crps": 1.378395, + "emos_mean_crps": 1.333721, + "legacy_mean_mae": 1.601905, + "emos_mean_mae": 1.565253, + "legacy_bucket_hit_rate": 0.428571, + "emos_bucket_hit_rate": 0.380952 + }, + "seoul": { + "samples": 22, + "legacy_mean_crps": 1.449956, + "emos_mean_crps": 1.264612, + "legacy_mean_mae": 1.817727, + "emos_mean_mae": 1.604882, + "legacy_bucket_hit_rate": 0.409091, + "emos_bucket_hit_rate": 0.454545 + }, + "shanghai": { + "samples": 23, + "legacy_mean_crps": 1.237937, + "emos_mean_crps": 1.196583, + "legacy_mean_mae": 1.458261, + "emos_mean_mae": 1.430445, + "legacy_bucket_hit_rate": 0.565217, + "emos_bucket_hit_rate": 0.608696 + }, + "shenzhen": { + "samples": 21, + "legacy_mean_crps": 0.481109, + "emos_mean_crps": 0.444093, + "legacy_mean_mae": 0.608095, + "emos_mean_mae": 0.581253, + "legacy_bucket_hit_rate": 0.619048, + "emos_bucket_hit_rate": 0.666667 + }, + "singapore": { + "samples": 22, + "legacy_mean_crps": 0.852942, + "emos_mean_crps": 0.758454, + "legacy_mean_mae": 1.020909, + "emos_mean_mae": 0.915982, + "legacy_bucket_hit_rate": 0.409091, + "emos_bucket_hit_rate": 0.454545 + }, + "taipei": { + "samples": 23, + "legacy_mean_crps": 0.801052, + "emos_mean_crps": 0.721468, + "legacy_mean_mae": 0.85087, + "emos_mean_mae": 0.814252, + "legacy_bucket_hit_rate": 0.478261, + "emos_bucket_hit_rate": 0.391304 + }, + "tel aviv": { + "samples": 22, + "legacy_mean_crps": 0.770763, + "emos_mean_crps": 0.672956, + "legacy_mean_mae": 0.904091, + "emos_mean_mae": 0.843671, + "legacy_bucket_hit_rate": 0.454545, + "emos_bucket_hit_rate": 0.5 + }, + "tokyo": { + "samples": 22, + "legacy_mean_crps": 0.651385, + "emos_mean_crps": 0.641684, + "legacy_mean_mae": 0.798182, + "emos_mean_mae": 0.787542, + "legacy_bucket_hit_rate": 0.545455, + "emos_bucket_hit_rate": 0.590909 + }, + "toronto": { + "samples": 22, + "legacy_mean_crps": 2.733044, + "emos_mean_crps": 2.714624, + "legacy_mean_mae": 2.981818, + "emos_mean_mae": 2.969098, + "legacy_bucket_hit_rate": 0.318182, + "emos_bucket_hit_rate": 0.409091 + }, + "warsaw": { + "samples": 23, + "legacy_mean_crps": 0.668776, + "emos_mean_crps": 0.665164, + "legacy_mean_mae": 0.796087, + "emos_mean_mae": 0.796958, + "legacy_bucket_hit_rate": 0.652174, + "emos_bucket_hit_rate": 0.608696 + }, + "wellington": { + "samples": 22, + "legacy_mean_crps": 0.731346, + "emos_mean_crps": 0.726977, + "legacy_mean_mae": 0.895, + "emos_mean_mae": 0.892558, + "legacy_bucket_hit_rate": 0.5, + "emos_bucket_hit_rate": 0.5 + }, + "wuhan": { + "samples": 20, + "legacy_mean_crps": 0.925443, + "emos_mean_crps": 0.897806, + "legacy_mean_mae": 1.0375, + "emos_mean_mae": 1.003133, + "legacy_bucket_hit_rate": 0.65, + "emos_bucket_hit_rate": 0.65 + } + } +} \ No newline at end of file diff --git a/artifacts/probability_calibration/default.backup-20260421122808.json b/artifacts/probability_calibration/default.backup-20260421122808.json new file mode 100644 index 00000000..a3796c3e --- /dev/null +++ b/artifacts/probability_calibration/default.backup-20260421122808.json @@ -0,0 +1,115 @@ +{ + "version": "emos-20260418192717", + "trained_at": "2026-04-18T19:27:17.348931+00:00", + "global": { + "mu": { + "intercept": 2.60074353, + "raw_mu_coef": 1.97765258, + "deb_coef": -1.37457626, + "ens_median_coef": 0.10693472, + "max_so_far_gap_coef": -0.58256593 + }, + "sigma": { + "intercept": 0.6373609, + "raw_sigma_coef": -1.41537225, + "spread_coef": 2.12997577, + "peak_flag_coef": -0.08633203, + "max_so_far_gap_coef": -0.00716383 + } + }, + "sigma_constraints": { + "min_ratio": 0.85, + "max_ratio": 1.35, + "absolute_min": 0.25, + "absolute_max": 3.0 + }, + "selection_guardrails": { + "max_mae_increase": 0.02, + "max_bucket_hit_drop": 0.01, + "max_bucket_brier_increase": 0.05 + }, + "blending": { + "alpha_mu": 0.05, + "alpha_sigma": 0.7 + }, + "cities": { + "tokyo": { + "samples": 5, + "mu_bias": 2.101101, + "sigma_scale": 2.0, + "confidence": 0.625 + }, + "ankara": { + "samples": 6, + "mu_bias": -0.523145, + "sigma_scale": 1.139849, + "confidence": 0.75 + }, + "istanbul": { + "samples": 3, + "mu_bias": -0.285183, + "sigma_scale": 0.550171, + "confidence": 0.375 + }, + "hong kong": { + "samples": 7, + "mu_bias": 3.234676, + "sigma_scale": 2.0, + "confidence": 0.875 + }, + "taipei": { + "samples": 5, + "mu_bias": 4.731246, + "sigma_scale": 2.0, + "confidence": 0.625 + }, + "busan": { + "samples": 3, + "mu_bias": 0.177772, + "sigma_scale": 2.0, + "confidence": 0.375 + }, + "seoul": { + "samples": 3, + "mu_bias": -3.070325, + "sigma_scale": 1.462196, + "confidence": 0.375 + }, + "milan": { + "samples": 3, + "mu_bias": -4.249866, + "sigma_scale": 2.0, + "confidence": 0.375 + }, + "shanghai": { + "samples": 3, + "mu_bias": 1.080039, + "sigma_scale": 1.638119, + "confidence": 0.375 + }, + "warsaw": { + "samples": 3, + "mu_bias": -1.68611, + "sigma_scale": 1.38114, + "confidence": 0.375 + } + }, + "metrics": { + "sample_count": 74, + "mean_crps": 3.181501, + "legacy_mean_crps": 3.474108, + "legacy_mean_mae": 3.679324, + "legacy_bucket_hit_rate": 0.5, + "legacy_bucket_brier": 0.855395, + "selected_mean_crps": 3.322469, + "selected_mean_mae": 3.618254, + "selected_bucket_hit_rate": 0.5, + "selected_bucket_brier": 0.863048, + "selected_score": 5.08158, + "legacy_score": 5.233586, + "filled_actual_from_history": 0, + "settlement_history_city_count": 30, + "legacy_archive_samples": 54 + }, + "source": "artifacts\\probability_calibration\\default.json" +} \ No newline at end of file diff --git a/artifacts/probability_calibration/default.json b/artifacts/probability_calibration/default.json index a3796c3e..ad104d58 100644 --- a/artifacts/probability_calibration/default.json +++ b/artifacts/probability_calibration/default.json @@ -1,20 +1,20 @@ { - "version": "emos-20260418192717", - "trained_at": "2026-04-18T19:27:17.348931+00:00", + "version": "emos-auto-20260421122743", + "trained_at": "2026-04-21T12:28:05.031165+00:00", "global": { "mu": { - "intercept": 2.60074353, - "raw_mu_coef": 1.97765258, - "deb_coef": -1.37457626, - "ens_median_coef": 0.10693472, - "max_so_far_gap_coef": -0.58256593 + "intercept": 0.34330438, + "raw_mu_coef": 0.48749629, + "deb_coef": 0.45733479, + "ens_median_coef": 0.05955297, + "max_so_far_gap_coef": -0.104935 }, "sigma": { - "intercept": 0.6373609, - "raw_sigma_coef": -1.41537225, - "spread_coef": 2.12997577, - "peak_flag_coef": -0.08633203, - "max_so_far_gap_coef": -0.00716383 + "intercept": -0.62765582, + "raw_sigma_coef": 0.38492261, + "spread_coef": 0.29095921, + "peak_flag_coef": 0.33966327, + "max_so_far_gap_coef": 0.08357376 } }, "sigma_constraints": { @@ -29,87 +29,333 @@ "max_bucket_brier_increase": 0.05 }, "blending": { - "alpha_mu": 0.05, - "alpha_sigma": 0.7 + "alpha_mu": 0.25, + "alpha_sigma": 1.0 }, "cities": { - "tokyo": { - "samples": 5, - "mu_bias": 2.101101, - "sigma_scale": 2.0, - "confidence": 0.625 + "istanbul": { + "samples": 20, + "mu_bias": -0.438949, + "sigma_scale": 0.553793, + "confidence": 1.0 + }, + "buenos aires": { + "samples": 22, + "mu_bias": -1.044971, + "sigma_scale": 1.389753, + "confidence": 1.0 }, "ankara": { - "samples": 6, - "mu_bias": -0.523145, - "sigma_scale": 1.139849, - "confidence": 0.75 + "samples": 23, + "mu_bias": 0.067406, + "sigma_scale": 1.270298, + "confidence": 1.0 }, - "istanbul": { - "samples": 3, - "mu_bias": -0.285183, - "sigma_scale": 0.550171, - "confidence": 0.375 - }, - "hong kong": { - "samples": 7, - "mu_bias": 3.234676, + "london": { + "samples": 22, + "mu_bias": -0.010833, "sigma_scale": 2.0, - "confidence": 0.875 + "confidence": 1.0 }, - "taipei": { + "paris": { + "samples": 22, + "mu_bias": -0.21677, + "sigma_scale": 1.095669, + "confidence": 1.0 + }, + "new york": { + "samples": 21, + "mu_bias": 1.072189, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "lagos": { + "samples": 10, + "mu_bias": -0.092732, + "sigma_scale": 1.558646, + "confidence": 1.0 + }, + "madrid": { + "samples": 22, + "mu_bias": -0.627862, + "sigma_scale": 1.761239, + "confidence": 1.0 + }, + "toronto": { + "samples": 22, + "mu_bias": -0.424101, + "sigma_scale": 1.326675, + "confidence": 1.0 + }, + "wellington": { + "samples": 22, + "mu_bias": -0.347796, + "sigma_scale": 0.893459, + "confidence": 1.0 + }, + "los angeles": { + "samples": 20, + "mu_bias": -0.659983, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "san francisco": { + "samples": 20, + "mu_bias": 0.941818, + "sigma_scale": 1.979403, + "confidence": 1.0 + }, + "aurora": { + "samples": 20, + "mu_bias": 2.040009, + "sigma_scale": 0.982492, + "confidence": 1.0 + }, + "austin": { + "samples": 20, + "mu_bias": 0.990023, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "houston": { + "samples": 20, + "mu_bias": 0.188787, + "sigma_scale": 1.819594, + "confidence": 1.0 + }, + "chicago": { + "samples": 21, + "mu_bias": 2.285098, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "dallas": { + "samples": 21, + "mu_bias": 0.258095, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "atlanta": { + "samples": 22, + "mu_bias": -2.093105, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "seattle": { + "samples": 21, + "mu_bias": -0.982188, + "sigma_scale": 1.695172, + "confidence": 1.0 + }, + "sao paulo": { + "samples": 22, + "mu_bias": -0.786242, + "sigma_scale": 1.427037, + "confidence": 1.0 + }, + "munich": { + "samples": 22, + "mu_bias": 0.096559, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "milan": { + "samples": 23, + "mu_bias": -0.045038, + "sigma_scale": 1.626331, + "confidence": 1.0 + }, + "warsaw": { + "samples": 23, + "mu_bias": -0.50853, + "sigma_scale": 1.239146, + "confidence": 1.0 + }, + "helsinki": { + "samples": 17, + "mu_bias": 0.442247, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "amsterdam": { + "samples": 17, + "mu_bias": 0.129889, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "panama city": { + "samples": 17, + "mu_bias": -0.040792, + "sigma_scale": 1.393259, + "confidence": 1.0 + }, + "cape town": { + "samples": 10, + "mu_bias": 0.507293, + "sigma_scale": 1.420004, + "confidence": 1.0 + }, + "miami": { + "samples": 22, + "mu_bias": -2.736316, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "manila": { "samples": 5, - "mu_bias": 4.731246, + "mu_bias": 1.772229, "sigma_scale": 2.0, "confidence": 0.625 }, - "busan": { - "samples": 3, - "mu_bias": 0.177772, - "sigma_scale": 2.0, - "confidence": 0.375 + "singapore": { + "samples": 22, + "mu_bias": 0.289639, + "sigma_scale": 1.124006, + "confidence": 1.0 + }, + "tokyo": { + "samples": 22, + "mu_bias": -0.332129, + "sigma_scale": 0.729483, + "confidence": 1.0 }, "seoul": { - "samples": 3, - "mu_bias": -3.070325, - "sigma_scale": 1.462196, - "confidence": 0.375 + "samples": 22, + "mu_bias": 1.6533, + "sigma_scale": 0.82648, + "confidence": 1.0 }, - "milan": { - "samples": 3, - "mu_bias": -4.249866, - "sigma_scale": 2.0, - "confidence": 0.375 + "busan": { + "samples": 16, + "mu_bias": 0.343239, + "sigma_scale": 1.886868, + "confidence": 1.0 + }, + "mexico city": { + "samples": 20, + "mu_bias": 0.087122, + "sigma_scale": 1.108884, + "confidence": 1.0 }, "shanghai": { - "samples": 3, - "mu_bias": 1.080039, - "sigma_scale": 1.638119, - "confidence": 0.375 + "samples": 23, + "mu_bias": 0.348022, + "sigma_scale": 0.810527, + "confidence": 1.0 }, - "warsaw": { - "samples": 3, - "mu_bias": -1.68611, - "sigma_scale": 1.38114, - "confidence": 0.375 + "beijing": { + "samples": 20, + "mu_bias": -0.288176, + "sigma_scale": 0.952972, + "confidence": 1.0 + }, + "shenzhen": { + "samples": 21, + "mu_bias": -0.006114, + "sigma_scale": 0.728494, + "confidence": 1.0 + }, + "wuhan": { + "samples": 20, + "mu_bias": -0.771154, + "sigma_scale": 1.30329, + "confidence": 1.0 + }, + "chengdu": { + "samples": 20, + "mu_bias": 0.445906, + "sigma_scale": 0.796989, + "confidence": 1.0 + }, + "chongqing": { + "samples": 20, + "mu_bias": -0.021301, + "sigma_scale": 1.106566, + "confidence": 1.0 + }, + "hong kong": { + "samples": 24, + "mu_bias": 0.37527, + "sigma_scale": 0.517756, + "confidence": 1.0 + }, + "taipei": { + "samples": 23, + "mu_bias": 0.600671, + "sigma_scale": 0.5, + "confidence": 1.0 + }, + "guangzhou": { + "samples": 6, + "mu_bias": 0.7082, + "sigma_scale": 1.975103, + "confidence": 0.75 + }, + "lau fau shan": { + "samples": 19, + "mu_bias": 0.41371, + "sigma_scale": 0.656578, + "confidence": 1.0 + }, + "kuala lumpur": { + "samples": 16, + "mu_bias": 0.499197, + "sigma_scale": 1.361171, + "confidence": 1.0 + }, + "jakarta": { + "samples": 16, + "mu_bias": 0.293537, + "sigma_scale": 0.751498, + "confidence": 1.0 + }, + "lucknow": { + "samples": 22, + "mu_bias": -1.593437, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "karachi": { + "samples": 5, + "mu_bias": -0.035595, + "sigma_scale": 0.588612, + "confidence": 0.625 + }, + "jeddah": { + "samples": 10, + "mu_bias": 0.516879, + "sigma_scale": 2.0, + "confidence": 1.0 + }, + "moscow": { + "samples": 20, + "mu_bias": -0.609602, + "sigma_scale": 0.783723, + "confidence": 1.0 + }, + "tel aviv": { + "samples": 22, + "mu_bias": 0.436837, + "sigma_scale": 1.46524, + "confidence": 1.0 } }, "metrics": { - "sample_count": 74, - "mean_crps": 3.181501, - "legacy_mean_crps": 3.474108, - "legacy_mean_mae": 3.679324, - "legacy_bucket_hit_rate": 0.5, - "legacy_bucket_brier": 0.855395, - "selected_mean_crps": 3.322469, - "selected_mean_mae": 3.618254, - "selected_bucket_hit_rate": 0.5, - "selected_bucket_brier": 0.863048, - "selected_score": 5.08158, - "legacy_score": 5.233586, + "sample_count": 978, + "mean_crps": 1.330733, + "legacy_mean_crps": 1.376936, + "legacy_mean_mae": 1.589438, + "legacy_bucket_hit_rate": 0.43865, + "legacy_bucket_brier": 0.824571, + "selected_mean_crps": 1.311763, + "selected_mean_mae": 1.553658, + "selected_bucket_hit_rate": 0.432515, + "selected_bucket_brier": 0.789817, + "selected_score": 2.910718, + "legacy_score": 2.996333, "filled_actual_from_history": 0, "settlement_history_city_count": 30, "legacy_archive_samples": 54 }, - "source": "artifacts\\probability_calibration\\default.json" + "source": "artifacts\\local_runtime\\probability_calibration\\candidates\\emos-auto-20260421122743\\default.json" } \ No newline at end of file diff --git a/frontend/components/dashboard/Dashboard.module.css b/frontend/components/dashboard/Dashboard.module.css index de6472fa..2bfe210c 100644 --- a/frontend/components/dashboard/Dashboard.module.css +++ b/frontend/components/dashboard/Dashboard.module.css @@ -1075,6 +1075,72 @@ border: 1px solid rgba(251, 113, 133, 0.26); } +.root :global(.prob-price-card) { + display: grid; + gap: 8px; + padding: 10px; + border: 1px solid rgba(34, 211, 238, 0.16); + border-radius: 8px; + background: rgba(8, 20, 32, 0.52); +} + +.root :global(.prob-price-head) { + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; +} + +.root :global(.prob-price-head span) { + color: var(--cyan); + font-size: 11px; + font-weight: 800; + text-transform: uppercase; + letter-spacing: 0.04em; +} + +.root :global(.prob-price-head strong) { + color: var(--text-primary); + font-size: 13px; + font-weight: 800; +} + +.root :global(.prob-price-grid) { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 8px; +} + +.root :global(.prob-price-grid > div) { + display: grid; + gap: 3px; + min-width: 0; + padding: 8px; + border: 1px solid rgba(148, 163, 184, 0.12); + border-radius: 8px; + background: rgba(15, 23, 42, 0.45); +} + +.root :global(.prob-price-grid span), +.root :global(.prob-price-grid em), +.root :global(.prob-price-card p) { + color: var(--text-muted); + font-size: 11px; + font-style: normal; + line-height: 1.35; +} + +.root :global(.prob-price-grid strong) { + color: var(--text-primary); + font-size: 13px; + font-weight: 800; + font-variant-numeric: tabular-nums; +} + +.root :global(.prob-price-card p) { + margin: 0; +} + .root :global(.prob-model-hint) { display: flex; align-items: center; @@ -2466,6 +2532,15 @@ .root :global(.marker-name) { font-size: 11px; } + + .root :global(.prob-price-grid) { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + + .root :global(.prob-price-head) { + align-items: flex-start; + flex-direction: column; + } } /* ── History Modal ── */ diff --git a/frontend/components/dashboard/PanelSections.tsx b/frontend/components/dashboard/PanelSections.tsx index c849c182..eacb9d45 100644 --- a/frontend/components/dashboard/PanelSections.tsx +++ b/frontend/components/dashboard/PanelSections.tsx @@ -278,6 +278,29 @@ function normalizeMarketProbability(value?: number | null) { return Math.max(0, Math.min(1, numeric)); } +function normalizeSignedProbability(value?: number | null) { + if (value == null) return null; + const numeric = Number(value); + if (!Number.isFinite(numeric)) return null; + if (Math.abs(numeric) > 1) return numeric / 100; + return numeric; +} + +function formatSignedPercent(value?: number | null, digits = 1) { + const normalized = normalizeSignedProbability(value); + if (normalized == null) return "--"; + const percent = normalized * 100; + const sign = percent > 0 ? "+" : ""; + return `${sign}${percent.toFixed(digits)}%`; +} + +function formatKellyFraction(value?: number | null) { + const normalized = normalizeSignedProbability(value); + if (normalized == null) return "--"; + if (normalized <= 0) return "0%"; + return `${(normalized * 100).toFixed(1)}%`; +} + function getMarketTopBuckets(scan?: MarketScan | null) { const buckets = Array.isArray(scan?.top_buckets) ? scan.top_buckets : []; if (!buckets.length) return []; @@ -657,6 +680,38 @@ export function ProbabilityDistribution({ const topProbabilityLabel = topProbability ? topProbability.label || `${topProbability.value}${detail.temp_symbol}` : null; + const priceAnalysis = marketScan?.price_analysis; + const yesPriceView = priceAnalysis?.yes; + const noPriceView = priceAnalysis?.no; + const preferredPriceView = + priceAnalysis?.best_side === "no" ? noPriceView : yesPriceView; + const preferredSideLabel = + priceAnalysis?.best_side === "no" + ? locale === "en-US" + ? "NO" + : "NO" + : locale === "en-US" + ? "YES" + : "YES"; + const hasPriceAnalysis = + Boolean(priceAnalysis?.available) && + (yesPriceView?.ask != null || noPriceView?.ask != null); + const lockEdge = normalizeSignedProbability(priceAnalysis?.lock?.edge); + const lockAvailable = Boolean(priceAnalysis?.lock?.available && lockEdge != null); + const quoteSource = + marketScan?.yes_token?.quote_source || + marketScan?.no_token?.quote_source || + null; + const quoteAgeMs = + marketScan?.yes_token?.quote_age_ms ?? marketScan?.no_token?.quote_age_ms; + const quoteSourceLabel = + quoteSource === "polymarket_ws" + ? locale === "en-US" + ? `WS live${quoteAgeMs != null ? ` · ${Math.max(0, Math.round(Number(quoteAgeMs) / 1000))}s` : ""}` + : `WS 实时${quoteAgeMs != null ? ` · ${Math.max(0, Math.round(Number(quoteAgeMs) / 1000))}秒` : ""}` + : locale === "en-US" + ? "CLOB fallback" + : "CLOB 兜底"; return (
@@ -702,6 +757,63 @@ export function ProbabilityDistribution({ : `市场仅作参考：该温度桶 ${marketYesText}`} )} + {hasPriceAnalysis && ( +
+
+ + {locale === "en-US" ? "Price reference" : "价格参考"} + + + {preferredPriceView?.edge != null + ? locale === "en-US" + ? `${preferredSideLabel} edge ${formatSignedPercent(preferredPriceView.edge)}` + : `${preferredSideLabel} 边际 ${formatSignedPercent(preferredPriceView.edge)}` + : locale === "en-US" + ? "Waiting for executable quote" + : "等待可执行报价"} + +
+
+
+ YES ask + {toPriceCents(yesPriceView?.ask) || "--"} + {formatSignedPercent(yesPriceView?.edge)} +
+
+ NO ask + {toPriceCents(noPriceView?.ask) || "--"} + {formatSignedPercent(noPriceView?.edge)} +
+
+ {locale === "en-US" ? "1/4 Kelly" : "1/4 Kelly"} + {formatKellyFraction(preferredPriceView?.quarter_kelly)} + {locale === "en-US" ? "cap required" : "需限额"} +
+
+ {locale === "en-US" ? "Lock" : "锁价"} + + {lockAvailable + ? formatSignedPercent(lockEdge) + : locale === "en-US" + ? "None" + : "无"} + + + {priceAnalysis?.lock?.ask_sum != null + ? locale === "en-US" + ? `sum ${toPriceCents(priceAnalysis.lock.ask_sum)}` + : `合计 ${toPriceCents(priceAnalysis.lock.ask_sum)}` + : "--"} + +
+
+

+ {locale === "en-US" + ? `Read-only sizing reference from executable quotes; it does not place orders. Source: ${quoteSourceLabel}.` + : `基于可执行盘口的只读仓位参考；系统不会下单。来源：${quoteSourceLabel}。`} +

+
+ )} {modelVoteHint && (
diff --git a/frontend/lib/dashboard-types.ts b/frontend/lib/dashboard-types.ts index 8b7c1c72..7962412a 100644 --- a/frontend/lib/dashboard-types.ts +++ b/frontend/lib/dashboard-types.ts @@ -289,6 +289,8 @@ export interface MarketToken { sell_price?: number | null; midpoint?: number | null; last_trade_price?: number | null; + quote_source?: string | null; + quote_age_ms?: number | null; } export interface MarketPrimary { @@ -314,11 +316,42 @@ export interface MarketTopBucket { yes_sell?: number | null; no_buy?: number | null; no_sell?: number | null; + quote_source?: string | null; + quote_age_ms?: number | null; slug?: string | null; question?: string | null; is_primary?: boolean; } +export interface MarketPriceSide { + side?: "yes" | "no" | string | null; + model_probability?: number | null; + ask?: number | null; + bid?: number | null; + edge?: number | null; + edge_percent?: number | null; + kelly_fraction?: number | null; + quarter_kelly?: number | null; +} + +export interface MarketPriceAnalysis { + available?: boolean; + source?: string | null; + model_probability?: number | null; + yes?: MarketPriceSide | null; + no?: MarketPriceSide | null; + best_side?: "yes" | "no" | string | null; + lock?: { + available?: boolean; + ask_sum?: number | null; + edge?: number | null; + } | null; + sell_side?: { + bid_sum?: number | null; + edge?: number | null; + } | null; +} + export interface MarketScan { available?: boolean; reason?: string | null; @@ -343,6 +376,7 @@ export interface MarketScan { last_trade_price?: number | null; liquidity?: number | null; volume?: number | null; + price_analysis?: MarketPriceAnalysis | null; sparkline?: number[]; top_buckets?: MarketTopBucket[] | null; recent_trades?: unknown[]; diff --git a/requirements.txt b/requirements.txt index 501dab9a..c2a5d98c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ lightgbm web3 fastapi uvicorn +websockets diff --git a/src/data_collection/polymarket_readonly.py b/src/data_collection/polymarket_readonly.py index efcbf26b..08a4a4ca 100644 --- a/src/data_collection/polymarket_readonly.py +++ b/src/data_collection/polymarket_readonly.py @@ -23,6 +23,7 @@ import httpx from loguru import logger from src.data_collection.city_registry import ALIASES, CITY_REGISTRY +from src.data_collection.polymarket_ws_cache import PolymarketWsQuoteCache try: from py_clob_client.client import ClobClient # type: ignore @@ -173,6 +174,16 @@ def _extract_price(value: Any) -> Optional[float]: return None +def _clamp_probability(value: Optional[float]) -> Optional[float]: + if value is None: + return None + if value < 0.0: + return 0.0 + if value > 1.0: + return 1.0 + return value + + def _extract_iso_date(value: Any) -> Optional[str]: if not value: return None @@ -394,6 +405,7 @@ class PolymarketReadOnlyLayer: self._lock = threading.Lock() self._clob_client: Any = None self._clob_unavailable_reason: Optional[str] = None + self._ws_quote_cache = PolymarketWsQuoteCache.from_env() def build_market_scan( self, @@ -430,6 +442,7 @@ class PolymarketReadOnlyLayer: "last_trade_price": None, "liquidity": None, "volume": None, + "price_analysis": None, "sparkline": fallback_sparkline or [], "top_buckets": [], "recent_trades": [], @@ -570,6 +583,8 @@ class PolymarketReadOnlyLayer: "sell_price": _extract_price(yes_prices.get("sell")), "midpoint": _extract_price(yes_prices.get("midpoint")), "last_trade_price": _extract_price(yes_prices.get("last_trade_price")), + "quote_source": yes_prices.get("quote_source"), + "quote_age_ms": _safe_int(yes_prices.get("quote_age_ms"), 0), "book": yes_prices.get("book"), } no_payload = { @@ -580,8 +595,17 @@ class PolymarketReadOnlyLayer: "sell_price": _extract_price(no_prices.get("sell")), "midpoint": _extract_price(no_prices.get("midpoint")), "last_trade_price": _extract_price(no_prices.get("last_trade_price")), + "quote_source": no_prices.get("quote_source"), + "quote_age_ms": _safe_int(no_prices.get("quote_age_ms"), 0), "book": no_prices.get("book"), } + price_analysis = self._build_price_analysis( + model_probability=model_probability, + yes_buy=_extract_price(yes_payload.get("buy_price")), + yes_sell=_extract_price(yes_payload.get("sell_price")), + no_buy=_extract_price(no_payload.get("buy_price")), + no_sell=_extract_price(no_payload.get("sell_price")), + ) sparkline_values: List[float] = [] for candidate in ( @@ -617,12 +641,15 @@ class PolymarketReadOnlyLayer: "last_trade_price": last_trade_price, "liquidity": liquidity, "volume": volume, + "price_analysis": price_analysis, "sparkline": sparkline_values, "top_buckets": top_buckets, "all_buckets": all_buckets, - "websocket": { - "market_url": market_url, - "asset_ids": [ + "websocket": { + "enabled": self._ws_quote_cache.enabled, + "status": self._ws_quote_cache.status(), + "market_url": market_url, + "asset_ids": [ token for token in [ yes_payload.get("token_id"), @@ -636,6 +663,112 @@ class PolymarketReadOnlyLayer: ) return scan + def _build_price_analysis( + self, + *, + model_probability: Optional[float], + yes_buy: Optional[float], + yes_sell: Optional[float], + no_buy: Optional[float], + no_sell: Optional[float], + ) -> Dict[str, Any]: + """Build read-only market price diagnostics. + + Polymarket CLOB naming is from the user's perspective: + BUY is the executable ask to buy that outcome, SELL is the executable bid. + Kelly here is a sizing reference only; no order execution is performed. + """ + p_yes = _clamp_probability(_safe_float(model_probability)) + p_no = _clamp_probability(1.0 - p_yes if p_yes is not None else None) + yes_ask = _clamp_probability(_safe_float(yes_buy)) + no_ask = _clamp_probability(_safe_float(no_buy)) + yes_bid = _clamp_probability(_safe_float(yes_sell)) + no_bid = _clamp_probability(_safe_float(no_sell)) + + yes = self._build_side_price_analysis("yes", p_yes, yes_ask, yes_bid) + no = self._build_side_price_analysis("no", p_no, no_ask, no_bid) + + ask_sum = None + lock_edge = None + lock_available = False + if yes_ask is not None and no_ask is not None: + ask_sum = yes_ask + no_ask + lock_edge = 1.0 - ask_sum + lock_available = lock_edge > 0 + + bid_sum = None + sell_side_edge = None + if yes_bid is not None and no_bid is not None: + bid_sum = yes_bid + no_bid + sell_side_edge = bid_sum - 1.0 + + best_side = None + side_rows = [ + row + for row in [yes, no] + if isinstance(row.get("edge"), (int, float)) + and isinstance(row.get("kelly_fraction"), (int, float)) + and row.get("kelly_fraction") > 0 + ] + if side_rows: + best_side = max( + side_rows, + key=lambda row: ( + float(row.get("edge") or 0.0), + float(row.get("kelly_fraction") or 0.0), + ), + ).get("side") + + return { + "available": any( + value is not None + for value in (yes_ask, no_ask, yes_bid, no_bid, p_yes) + ), + "source": "polymarket_clob_orderbook", + "model_probability": p_yes, + "yes": yes, + "no": no, + "best_side": best_side, + "lock": { + "available": lock_available, + "ask_sum": ask_sum, + "edge": lock_edge, + }, + "sell_side": { + "bid_sum": bid_sum, + "edge": sell_side_edge, + }, + } + + def _build_side_price_analysis( + self, + side: str, + probability: Optional[float], + ask: Optional[float], + bid: Optional[float], + ) -> Dict[str, Any]: + edge = None + kelly_fraction = None + if probability is not None and ask is not None: + edge = probability - ask + if 0.0 < ask < 1.0: + kelly_fraction = edge / (1.0 - ask) + + return { + "side": side, + "model_probability": probability, + "ask": ask, + "bid": bid, + "edge": edge, + "edge_percent": edge * 100.0 if edge is not None else None, + "kelly_fraction": kelly_fraction, + "quarter_kelly": ( + max(0.0, kelly_fraction) / 4.0 + if kelly_fraction is not None + else None + ), + } + def _market_trade_state(self, market: Dict[str, Any]) -> Dict[str, Any]: active = _safe_bool(market.get("active")) closed_raw = _safe_bool(market.get("closed")) @@ -1193,6 +1326,11 @@ class PolymarketReadOnlyLayer: if not token_id: return {} + self._ws_quote_cache.subscribe([token_id]) + ws_data = self._ws_quote_cache.get_market_data(token_id) + if ws_data is not None: + return ws_data + now = time.time() with self._lock: cached = self._price_cache.get(token_id) @@ -1473,6 +1611,8 @@ class PolymarketReadOnlyLayer: "yes_sell": yes_sell, "no_buy": no_buy, "no_sell": no_sell, + "quote_source": yes_prices.get("quote_source"), + "quote_age_ms": _safe_int(yes_prices.get("quote_age_ms"), 0), "slug": market_slug or None, "question": market.get("question") or market.get("title"), "is_primary": bool( diff --git a/src/data_collection/polymarket_ws_cache.py b/src/data_collection/polymarket_ws_cache.py new file mode 100644 index 00000000..fcfb287c --- /dev/null +++ b/src/data_collection/polymarket_ws_cache.py @@ -0,0 +1,404 @@ +""" +Read-only Polymarket market WebSocket quote cache. + +The cache subscribes to public market-channel asset ids and stores executable +best bid / ask updates. It is deliberately optional: callers should keep REST +or CLOB polling as a fallback when the WebSocket client is unavailable. +""" + +from __future__ import annotations + +import asyncio +import json +import math +import os +import threading +import time +from typing import Any, Dict, Iterable, Optional, Set + +from loguru import logger + + +def _safe_float(value: Any) -> Optional[float]: + if value is None: + return None + try: + if isinstance(value, str): + value = value.strip() + if not value: + return None + numeric = float(value) + if math.isnan(numeric) or math.isinf(numeric): + return None + return numeric + except Exception: + return None + + +def _env_bool(name: str, default: bool = False) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +class PolymarketWsQuoteCache: + def __init__( + self, + *, + enabled: bool = False, + endpoint: Optional[str] = None, + quote_ttl_sec: int = 8, + max_assets: int = 256, + reconnect_delay_sec: float = 3.0, + ) -> None: + self.enabled = enabled + self.endpoint = ( + endpoint + or os.getenv( + "POLYMARKET_WS_MARKET_URL", + "wss://ws-subscriptions-clob.polymarket.com/ws/market", + ) + or "" + ).strip() + self.quote_ttl_sec = max(1, int(quote_ttl_sec or 8)) + self.max_assets = max(1, int(max_assets or 256)) + self.reconnect_delay_sec = max(0.5, float(reconnect_delay_sec or 3.0)) + + self._desired_assets: Set[str] = set() + self._quotes: Dict[str, Dict[str, Any]] = {} + self._lock = threading.Lock() + self._thread: Optional[threading.Thread] = None + self._stop_event = threading.Event() + self._started = False + self._last_error: Optional[str] = None + self._last_connected_at: Optional[float] = None + self._last_message_at: Optional[float] = None + + @classmethod + def from_env(cls) -> "PolymarketWsQuoteCache": + return cls( + enabled=_env_bool("POLYMARKET_WS_PRICE_ENABLED", False), + endpoint=os.getenv("POLYMARKET_WS_MARKET_URL"), + quote_ttl_sec=int(os.getenv("POLYMARKET_WS_QUOTE_TTL_SEC", "8")), + max_assets=int(os.getenv("POLYMARKET_WS_MAX_ASSETS", "256")), + reconnect_delay_sec=float( + os.getenv("POLYMARKET_WS_RECONNECT_DELAY_SEC", "3") + ), + ) + + def start(self) -> None: + if not self.enabled or not self.endpoint: + return + with self._lock: + if self._started: + return + self._started = True + self._thread = threading.Thread( + target=self._thread_main, + name="polymarket-ws-quotes", + daemon=True, + ) + self._thread.start() + + def stop(self) -> None: + self._stop_event.set() + + def subscribe(self, asset_ids: Iterable[Any]) -> None: + if not self.enabled: + return + normalized = [] + for asset_id in asset_ids: + text = str(asset_id or "").strip() + if text: + normalized.append(text) + if not normalized: + return + + with self._lock: + remaining = self.max_assets - len(self._desired_assets) + for asset_id in normalized: + if asset_id in self._desired_assets: + continue + if remaining <= 0: + break + self._desired_assets.add(asset_id) + remaining -= 1 + self.start() + + def get_market_data(self, asset_id: Any) -> Optional[Dict[str, Any]]: + quote = self.get_quote(asset_id) + if not quote: + return None + + best_bid = _safe_float(quote.get("best_bid")) + best_ask = _safe_float(quote.get("best_ask")) + if best_bid is None and best_ask is None: + return None + + midpoint = None + if best_bid is not None and best_ask is not None: + midpoint = (best_bid + best_ask) / 2.0 + + age_ms = int((time.time() - float(quote.get("t") or time.time())) * 1000) + return { + "buy": best_ask, + "sell": best_bid, + "midpoint": midpoint, + "last_trade_price": _safe_float(quote.get("last_trade_price")), + "book": { + "best_bid": best_bid, + "best_ask": best_ask, + "bid_levels": [[best_bid, 0.0]] if best_bid is not None else [], + "ask_levels": [[best_ask, 0.0]] if best_ask is not None else [], + }, + "book_liquidity": None, + "quote_source": "polymarket_ws", + "quote_age_ms": age_ms, + } + + def get_quote(self, asset_id: Any) -> Optional[Dict[str, Any]]: + text = str(asset_id or "").strip() + if not text: + return None + now = time.time() + with self._lock: + quote = self._quotes.get(text) + if not quote: + return None + if now - float(quote.get("t") or 0.0) > self.quote_ttl_sec: + return None + return dict(quote) + + def status(self) -> Dict[str, Any]: + with self._lock: + return { + "enabled": self.enabled, + "started": self._started, + "endpoint": self.endpoint, + "asset_count": len(self._desired_assets), + "quote_count": len(self._quotes), + "last_error": self._last_error, + "last_connected_at": self._last_connected_at, + "last_message_at": self._last_message_at, + } + + def _thread_main(self) -> None: + try: + asyncio.run(self._run_forever()) + except Exception as exc: # pragma: no cover - defensive thread guard + with self._lock: + self._last_error = str(exc) + logger.warning(f"Polymarket WS quote cache stopped: {exc}") + + async def _run_forever(self) -> None: + try: + import websockets # type: ignore + except Exception as exc: + with self._lock: + self._last_error = f"websockets import failed: {exc}" + logger.warning(self._last_error) + return + + while not self._stop_event.is_set(): + try: + async with websockets.connect( + self.endpoint, + ping_interval=None, + close_timeout=2, + ) as ws: + with self._lock: + self._last_connected_at = time.time() + self._last_error = None + subscribed: Set[str] = set() + last_ping = 0.0 + + while not self._stop_event.is_set(): + desired = self._snapshot_assets() + missing = desired - subscribed + if missing: + await self._send_subscription( + ws, + missing, + initial=not subscribed, + ) + subscribed.update(missing) + + now = time.time() + if now - last_ping >= 10: + await ws.send(json.dumps({})) + last_ping = now + + try: + raw = await asyncio.wait_for(ws.recv(), timeout=1.0) + except asyncio.TimeoutError: + continue + self._handle_message(raw) + except Exception as exc: + with self._lock: + self._last_error = str(exc) + logger.warning(f"Polymarket WS reconnecting after error: {exc}") + await asyncio.sleep(self.reconnect_delay_sec) + + def _snapshot_assets(self) -> Set[str]: + with self._lock: + return set(self._desired_assets) + + async def _send_subscription( + self, + ws: Any, + asset_ids: Iterable[str], + *, + initial: bool, + ) -> None: + batch = [asset_id for asset_id in asset_ids if asset_id] + if not batch: + return + payload = { + "assets_ids": batch, + "custom_feature_enabled": True, + } + if initial: + payload["type"] = "market" + else: + payload["operation"] = "subscribe" + await ws.send(json.dumps(payload)) + + def _handle_message(self, raw: Any) -> None: + if raw in (None, "", "PONG"): + return + try: + payload = json.loads(raw) if isinstance(raw, str) else raw + except Exception: + return + + if isinstance(payload, list): + for item in payload: + self._handle_event(item) + return + self._handle_event(payload) + + def _handle_event(self, event: Any) -> None: + if not isinstance(event, dict): + return + if "event_type" in event: + event_type = str(event.get("event_type") or "").strip().lower() + else: + event_type = str(event.get("type") or "").strip().lower() + + if event_type in { + "best_bid_ask", + "price_change", + "book", + "last_trade_price", + }: + self._handle_quote_event(event_type, event) + + def _handle_quote_event(self, event_type: str, event: Dict[str, Any]) -> None: + candidates = ( + event.get("price_changes") + or event.get("changes") + or event.get("assets") + or event.get("data") + ) + if isinstance(candidates, list): + for item in candidates: + if isinstance(item, dict): + self._upsert_quote(event_type, item, parent=event) + return + self._upsert_quote(event_type, event, parent=event) + + def _upsert_quote( + self, + event_type: str, + item: Dict[str, Any], + *, + parent: Dict[str, Any], + ) -> None: + asset_id = str( + item.get("asset_id") + or item.get("assetId") + or item.get("token_id") + or item.get("tokenId") + or parent.get("asset_id") + or parent.get("assetId") + or "" + ).strip() + if not asset_id: + return + + best_bid = ( + _safe_float(item.get("best_bid")) + or _safe_float(item.get("bid")) + or _safe_float(item.get("bestBid")) + ) + best_ask = ( + _safe_float(item.get("best_ask")) + or _safe_float(item.get("ask")) + or _safe_float(item.get("bestAsk")) + ) + if event_type == "book": + parsed_bid, parsed_ask = self._extract_book_top(item) + best_bid = best_bid if best_bid is not None else parsed_bid + best_ask = best_ask if best_ask is not None else parsed_ask + price = _safe_float(item.get("price")) + side = str(item.get("side") or "").strip().upper() + if event_type == "price_change" and price is not None: + if side == "BUY": + best_ask = price + elif side == "SELL": + best_bid = price + + last_trade = ( + _safe_float(item.get("last_trade_price")) + or _safe_float(item.get("lastTradePrice")) + or (price if event_type == "last_trade_price" else None) + ) + + now = time.time() + with self._lock: + previous = dict(self._quotes.get(asset_id) or {}) + if best_bid is not None: + previous["best_bid"] = best_bid + if best_ask is not None: + previous["best_ask"] = best_ask + if last_trade is not None: + previous["last_trade_price"] = last_trade + previous["asset_id"] = asset_id + previous["event_type"] = event_type + previous["t"] = now + self._quotes[asset_id] = previous + self._last_message_at = now + + def _extract_book_top( + self, + payload: Dict[str, Any], + ) -> tuple[Optional[float], Optional[float]]: + best_bid = None + best_ask = None + + bids = payload.get("bids") + if isinstance(bids, list): + for item in bids: + price = self._extract_level_price(item) + if price is None: + continue + best_bid = price if best_bid is None else max(best_bid, price) + + asks = payload.get("asks") + if isinstance(asks, list): + for item in asks: + price = self._extract_level_price(item) + if price is None: + continue + best_ask = price if best_ask is None else min(best_ask, price) + + return best_bid, best_ask + + @staticmethod + def _extract_level_price(level: Any) -> Optional[float]: + if isinstance(level, dict): + return _safe_float(level.get("price")) + if isinstance(level, (list, tuple)) and level: + return _safe_float(level[0]) + return None diff --git a/tests/test_polymarket_readonly.py b/tests/test_polymarket_readonly.py index a810e548..78e4c4dc 100644 --- a/tests/test_polymarket_readonly.py +++ b/tests/test_polymarket_readonly.py @@ -61,6 +61,62 @@ def test_fetch_token_market_data_prefers_orderbook_executable_prices(): assert data["last_trade_price"] == 0.49 +def test_get_token_market_data_prefers_fresh_ws_cache(): + layer = PolymarketReadOnlyLayer() + + class FakeWsCache: + enabled = True + + def subscribe(self, asset_ids): + self.asset_ids = list(asset_ids) + + @staticmethod + def get_market_data(_token_id): + return { + "buy": 0.33, + "sell": 0.31, + "midpoint": 0.32, + "quote_source": "polymarket_ws", + "quote_age_ms": 80, + } + + layer._ws_quote_cache = FakeWsCache() + layer._fetch_token_market_data = lambda _token_id: {"buy": 0.99} + + data = layer._get_token_market_data("token-1") + + assert data["buy"] == 0.33 + assert data["sell"] == 0.31 + assert data["quote_source"] == "polymarket_ws" + + +def test_price_analysis_computes_edge_kelly_and_lock(): + layer = PolymarketReadOnlyLayer() + + analysis = layer._build_price_analysis( + model_probability=0.62, + yes_buy=0.52, + yes_sell=0.50, + no_buy=0.45, + no_sell=0.43, + ) + + assert analysis["available"] is True + assert abs(analysis["yes"]["edge"] - 0.10) < 0.000001 + assert round(analysis["yes"]["kelly_fraction"], 6) == round( + (0.62 - 0.52) / (1.0 - 0.52), + 6, + ) + assert round(analysis["yes"]["quarter_kelly"], 6) == round( + ((0.62 - 0.52) / (1.0 - 0.52)) / 4.0, + 6, + ) + assert abs(analysis["no"]["edge"] - -0.07) < 0.000001 + assert analysis["lock"]["available"] is True + assert round(analysis["lock"]["edge"], 6) == 0.03 + assert analysis["best_side"] == "yes" + + def test_build_top_temperature_buckets_dedupes_same_temperature(): layer = PolymarketReadOnlyLayer() diff --git a/tests/test_polymarket_ws_cache.py b/tests/test_polymarket_ws_cache.py new file mode 100644 index 00000000..f232b842 --- /dev/null +++ b/tests/test_polymarket_ws_cache.py @@ -0,0 +1,124 @@ +import asyncio +import json +import time + +from src.data_collection.polymarket_ws_cache import PolymarketWsQuoteCache + + +def test_ws_cache_parses_best_bid_ask_event(): + cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=30) + + cache.subscribe(["asset-1"]) + cache._handle_message( + { + "event_type": "best_bid_ask", + "asset_id": "asset-1", + "best_bid": "0.41", + "best_ask": "0.44", + } + ) + + data = cache.get_market_data("asset-1") + + assert data is not None + assert data["sell"] == 0.41 + assert data["buy"] == 0.44 + assert data["midpoint"] == 0.425 + assert data["quote_source"] == "polymarket_ws" + + +def test_ws_cache_ignores_stale_quotes(): + cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=1) + cache._quotes["asset-1"] = { + "asset_id": "asset-1", + "best_bid": 0.41, + "best_ask": 0.44, + "t": time.time() - 10, + } + + assert cache.get_market_data("asset-1") is None + + +def test_ws_cache_parses_price_change_side_updates(): + cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=30) + + cache._handle_message( + { + "event_type": "price_change", + "changes": [ + { + "asset_id": "asset-1", + "side": "BUY", + "price": "0.48", + }, + { + "asset_id": "asset-1", + "side": "SELL", + "price": "0.45", + }, + ], + } + ) + + data = cache.get_market_data("asset-1") + + assert data is not None + assert data["buy"] == 0.48 + assert data["sell"] == 0.45 + + +def test_ws_cache_parses_price_changes_key_and_book_event(): + cache = PolymarketWsQuoteCache(enabled=True, quote_ttl_sec=30) + + cache._handle_message( + { + "event_type": "book", + "asset_id": "asset-1", + "bids": [{"price": "0.42"}, {"price": "0.44"}], + "asks": [{"price": "0.51"}, {"price": "0.49"}], + } + ) + cache._handle_message( + { + "event_type": "price_change", + "price_changes": [ + { + "asset_id": "asset-1", + "side": "BUY", + "price": "0.48", + } + ], + } + ) + + data = cache.get_market_data("asset-1") + + assert data is not None + assert data["sell"] == 0.44 + assert data["buy"] == 0.48 + + +def test_ws_cache_subscription_payloads_match_market_channel_shape(): + class FakeWs: + def __init__(self): + self.messages = [] + + async def send(self, payload): + self.messages.append(json.loads(payload)) + + cache = PolymarketWsQuoteCache(enabled=True) + ws = FakeWs() + + asyncio.run(cache._send_subscription(ws, ["asset-1"], initial=True)) + asyncio.run(cache._send_subscription(ws, ["asset-2"], initial=False)) + + assert ws.messages[0] == { + "type": "market", + "assets_ids": ["asset-1"], + "custom_feature_enabled": True, + } + assert ws.messages[1] == { + "operation": "subscribe", + "assets_ids": ["asset-2"], + "custom_feature_enabled": True, + }