From 611e2d900985ecb3942b3379a2dc9b55c03ec680 Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Thu, 16 Apr 2026 01:05:43 +0800 Subject: [PATCH] Expand calibration samples and extend training retention --- .../probability_calibration/default.json | 113 ++++--- .../evaluation_report.json | 297 ++++++++++++++---- scripts/evaluate_probability_calibration.py | 17 +- scripts/fit_probability_calibration.py | 54 ++++ src/analysis/deb_algorithm.py | 6 +- 5 files changed, 384 insertions(+), 103 deletions(-) diff --git a/artifacts/probability_calibration/default.json b/artifacts/probability_calibration/default.json index 138bb4a9..d41b189c 100644 --- a/artifacts/probability_calibration/default.json +++ b/artifacts/probability_calibration/default.json @@ -1,20 +1,20 @@ { - "version": "emos-20260415164136", - "trained_at": "2026-04-15T16:41:36.704829+00:00", + "version": "emos-20260415170349", + "trained_at": "2026-04-15T17:03:49.122865+00:00", "global": { "mu": { - "intercept": -0.39094186, - "raw_mu_coef": 0.27476248, - "deb_coef": 0.91159557, - "ens_median_coef": -0.16959018, - "max_so_far_gap_coef": -0.94232523 + "intercept": 2.45853035, + "raw_mu_coef": 2.03689031, + "deb_coef": -1.41207502, + "ens_median_coef": 0.08616811, + "max_so_far_gap_coef": -0.5886898 }, "sigma": { - "intercept": -1.56485068, - "raw_sigma_coef": 0.12930442, - "spread_coef": 0.15947819, - "peak_flag_coef": 0.19328621, - "max_so_far_gap_coef": -0.08469752 + "intercept": 0.61260118, + "raw_sigma_coef": -1.48747774, + "spread_coef": 2.24562634, + "peak_flag_coef": -0.15906747, + "max_so_far_gap_coef": -0.01835251 } }, "sigma_constraints": { @@ -29,56 +29,81 @@ "max_bucket_brier_increase": 0.05 }, "blending": { - "alpha_mu": 1.0, - "alpha_sigma": 0.5 + "alpha_mu": 0.05, + "alpha_sigma": 1.0 }, "cities": { "tokyo": { - "samples": 4, - "mu_bias": 0.211244, - "sigma_scale": 0.5, - "confidence": 0.5 + "samples": 5, + "mu_bias": 2.223946, + "sigma_scale": 2.0, + "confidence": 0.625 }, "ankara": { - "samples": 11, - "mu_bias": 0.027471, - "sigma_scale": 0.5, - "confidence": 1.0 + "samples": 6, + "mu_bias": -0.396953, + "sigma_scale": 1.094474, + "confidence": 0.75 }, "istanbul": { - "samples": 12, - "mu_bias": 0.048913, + "samples": 3, + "mu_bias": -0.163153, "sigma_scale": 0.5, - "confidence": 1.0 + "confidence": 0.375 }, "hong kong": { - "samples": 12, - "mu_bias": -0.026823, - "sigma_scale": 1.020494, - "confidence": 1.0 + "samples": 7, + "mu_bias": 3.267553, + "sigma_scale": 2.0, + "confidence": 0.875 }, "taipei": { "samples": 5, - "mu_bias": 0.192719, - "sigma_scale": 0.5, + "mu_bias": 4.80044, + "sigma_scale": 2.0, "confidence": 0.625 + }, + "seoul": { + "samples": 3, + "mu_bias": -3.080962, + "sigma_scale": 1.437353, + "confidence": 0.375 + }, + "milan": { + "samples": 3, + "mu_bias": -4.20255, + "sigma_scale": 2.0, + "confidence": 0.375 + }, + "shanghai": { + "samples": 3, + "mu_bias": 1.185335, + "sigma_scale": 1.68386, + "confidence": 0.375 + }, + "warsaw": { + "samples": 3, + "mu_bias": -1.574246, + "sigma_scale": 1.343402, + "confidence": 0.375 } }, "metrics": { - "sample_count": 50, - "mean_crps": 0.278798, - "legacy_mean_crps": 1.26525, - "legacy_mean_mae": 1.4674, - "legacy_bucket_hit_rate": 0.54, - "legacy_bucket_brier": 0.744518, - "selected_mean_crps": 0.278798, - "selected_mean_mae": 0.248457, - "selected_bucket_hit_rate": 0.82, - "selected_bucket_brier": 0.369542, - "selected_score": 0.8508, - "legacy_score": 2.660378, + "sample_count": 71, + "mean_crps": 3.226613, + "legacy_mean_crps": 3.608885, + "legacy_mean_mae": 3.81507, + "legacy_bucket_hit_rate": 0.492958, + "legacy_bucket_brier": 0.873776, + "selected_mean_crps": 3.44742, + "selected_mean_mae": 3.749078, + "selected_bucket_hit_rate": 0.492958, + "selected_bucket_brier": 0.878621, + "selected_score": 5.241857, + "legacy_score": 5.406287, "filled_actual_from_history": 0, - "settlement_history_city_count": 30 + "settlement_history_city_count": 30, + "legacy_archive_samples": 54 }, "source": "artifacts\\probability_calibration\\default.json" } \ No newline at end of file diff --git a/artifacts/probability_calibration/evaluation_report.json b/artifacts/probability_calibration/evaluation_report.json index 46cb1df6..9fba7f43 100644 --- a/artifacts/probability_calibration/evaluation_report.json +++ b/artifacts/probability_calibration/evaluation_report.json @@ -1,95 +1,284 @@ { "summary": { - "sample_count": 17, + "sample_count": 71, "filled_actual_from_history": 0, "legacy": { - "mean_crps": 1.095786, - "mean_mae": 1.243529, - "bucket_hit_rate": 0.588235 + "mean_crps": 3.608885, + "mean_mae": 3.81507, + "bucket_hit_rate": 0.492958 }, "emos": { - "mean_crps": 1.257578, - "mean_mae": 1.545328, - "bucket_hit_rate": 0.352941 + "mean_crps": 3.881798, + "mean_mae": 4.378282, + "bucket_hit_rate": 0.225352 }, "delta": { - "crps": 0.161791, - "mae": 0.301798, - "bucket_hit_rate": -0.235294 + "crps": 0.272913, + "mae": 0.563211, + "bucket_hit_rate": -0.267606 } }, "by_city": { "ankara": { - "samples": 3, - "legacy_mean_crps": 2.707176, - "emos_mean_crps": 2.632737, - "legacy_mean_mae": 3.0, - "emos_mean_mae": 2.90439, - "legacy_bucket_hit_rate": 0.333333, + "samples": 6, + "legacy_mean_crps": 1.365135, + "emos_mean_crps": 1.454394, + "legacy_mean_mae": 1.466667, + "emos_mean_mae": 1.781865, + "legacy_bucket_hit_rate": 0.666667, "emos_bucket_hit_rate": 0.333333 }, + "atlanta": { + "samples": 2, + "legacy_mean_crps": 30.449382, + "emos_mean_crps": 30.2136, + "legacy_mean_mae": 32.015, + "emos_mean_mae": 31.431544, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "buenos aires": { + "samples": 2, + "legacy_mean_crps": 9.113412, + "emos_mean_crps": 9.25091, + "legacy_mean_mae": 10.27, + "emos_mean_mae": 10.234011, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, "busan": { "samples": 1, - "legacy_mean_crps": 0.264698, - "emos_mean_crps": 0.797946, + "legacy_mean_crps": 0.451701, + "emos_mean_crps": 1.176676, "legacy_mean_mae": 0.3, - "emos_mean_mae": 1.213185, + "emos_mean_mae": 1.887548, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 0.0 }, "chengdu": { "samples": 1, - "legacy_mean_crps": 0.218827, - "emos_mean_crps": 0.321502, + "legacy_mean_crps": 0.306469, + "emos_mean_crps": 0.555327, "legacy_mean_mae": 0.3, - "emos_mean_mae": 0.539971, + "emos_mean_mae": 0.929502, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.0 + }, + "chicago": { + "samples": 1, + "legacy_mean_crps": 1.250268, + "emos_mean_crps": 2.090175, + "legacy_mean_mae": 0.0, + "emos_mean_mae": 3.396557, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.0 + }, + "dallas": { + "samples": 1, + "legacy_mean_crps": 2.173363, + "emos_mean_crps": 1.05572, + "legacy_mean_mae": 0.0, + "emos_mean_mae": 1.653475, "legacy_bucket_hit_rate": 1.0, "emos_bucket_hit_rate": 0.0 }, "hong kong": { - "samples": 3, - "legacy_mean_crps": 0.321946, - "emos_mean_crps": 0.419198, - "legacy_mean_mae": 0.27, - "emos_mean_mae": 0.673605, - "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.666667 + "samples": 7, + "legacy_mean_crps": 0.706099, + "emos_mean_crps": 1.573367, + "legacy_mean_mae": 0.657143, + "emos_mean_mae": 1.88803, + "legacy_bucket_hit_rate": 0.714286, + "emos_bucket_hit_rate": 0.142857 }, "istanbul": { "samples": 3, - "legacy_mean_crps": 1.308328, - "emos_mean_crps": 1.231189, - "legacy_mean_mae": 1.38, - "emos_mean_mae": 1.211693, - "legacy_bucket_hit_rate": 0.666667, + "legacy_mean_crps": 1.264103, + "emos_mean_crps": 1.266844, + "legacy_mean_mae": 1.5, + "emos_mean_mae": 1.415758, + "legacy_bucket_hit_rate": 0.333333, "emos_bucket_hit_rate": 0.666667 }, - "seoul": { - "samples": 1, - "legacy_mean_crps": 0.208763, - "emos_mean_crps": 2.708973, - "legacy_mean_mae": 0.3, - "emos_mean_mae": 3.02868, - "legacy_bucket_hit_rate": 1.0, - "emos_bucket_hit_rate": 0.0 - }, - "taipei": { + "london": { "samples": 2, - "legacy_mean_crps": 0.767971, - "emos_mean_crps": 1.205469, - "legacy_mean_mae": 1.195, - "emos_mean_mae": 1.937374, + "legacy_mean_crps": 3.885033, + "emos_mean_crps": 4.057521, + "legacy_mean_mae": 4.135, + "emos_mean_mae": 4.313103, "legacy_bucket_hit_rate": 0.0, "emos_bucket_hit_rate": 0.0 }, - "tokyo": { + "lucknow": { + "samples": 2, + "legacy_mean_crps": 2.487193, + "emos_mean_crps": 2.021264, + "legacy_mean_mae": 3.205, + "emos_mean_mae": 2.47462, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.5 + }, + "madrid": { + "samples": 2, + "legacy_mean_crps": 6.27726, + "emos_mean_crps": 6.588692, + "legacy_mean_mae": 7.33, + "emos_mean_mae": 7.486713, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "miami": { + "samples": 2, + "legacy_mean_crps": 28.637631, + "emos_mean_crps": 28.800796, + "legacy_mean_mae": 30.175, + "emos_mean_mae": 30.107593, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "milan": { "samples": 3, - "legacy_mean_crps": 1.129263, - "emos_mean_crps": 0.763363, - "legacy_mean_mae": 1.3, - "emos_mean_mae": 1.081641, - "legacy_bucket_hit_rate": 0.333333, + "legacy_mean_crps": 4.401392, + "emos_mean_crps": 4.947416, + "legacy_mean_mae": 4.06, + "emos_mean_mae": 5.6958, + "legacy_bucket_hit_rate": 0.666667, "emos_bucket_hit_rate": 0.333333 + }, + "munich": { + "samples": 2, + "legacy_mean_crps": 3.145192, + "emos_mean_crps": 3.200946, + "legacy_mean_mae": 3.64, + "emos_mean_mae": 3.634475, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "new york": { + "samples": 1, + "legacy_mean_crps": 3.692845, + "emos_mean_crps": 3.881457, + "legacy_mean_mae": 4.94, + "emos_mean_mae": 4.954702, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "paris": { + "samples": 2, + "legacy_mean_crps": 4.013782, + "emos_mean_crps": 4.130614, + "legacy_mean_mae": 4.265, + "emos_mean_mae": 4.314216, + "legacy_bucket_hit_rate": 0.5, + "emos_bucket_hit_rate": 0.5 + }, + "sao paulo": { + "samples": 2, + "legacy_mean_crps": 5.540967, + "emos_mean_crps": 5.483258, + "legacy_mean_mae": 6.57, + "emos_mean_mae": 6.358434, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "seattle": { + "samples": 1, + "legacy_mean_crps": 0.315488, + "emos_mean_crps": 0.271998, + "legacy_mean_mae": 0.0, + "emos_mean_mae": 0.105037, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 1.0 + }, + "seoul": { + "samples": 3, + "legacy_mean_crps": 0.508331, + "emos_mean_crps": 1.718042, + "legacy_mean_mae": 0.2, + "emos_mean_mae": 2.57121, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.0 + }, + "shanghai": { + "samples": 3, + "legacy_mean_crps": 0.299116, + "emos_mean_crps": 0.553444, + "legacy_mean_mae": 0.1, + "emos_mean_mae": 0.71746, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.666667 + }, + "shenzhen": { + "samples": 1, + "legacy_mean_crps": 1.198351, + "emos_mean_crps": 0.507942, + "legacy_mean_mae": 1.3, + "emos_mean_mae": 0.644561, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "singapore": { + "samples": 2, + "legacy_mean_crps": 0.281993, + "emos_mean_crps": 0.877023, + "legacy_mean_mae": 0.15, + "emos_mean_mae": 1.142765, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.5 + }, + "taipei": { + "samples": 5, + "legacy_mean_crps": 0.950739, + "emos_mean_crps": 1.257972, + "legacy_mean_mae": 0.94, + "emos_mean_mae": 1.687014, + "legacy_bucket_hit_rate": 0.6, + "emos_bucket_hit_rate": 0.2 + }, + "tel aviv": { + "samples": 2, + "legacy_mean_crps": 0.446758, + "emos_mean_crps": 0.482341, + "legacy_mean_mae": 0.3, + "emos_mean_mae": 0.62659, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.5 + }, + "tokyo": { + "samples": 5, + "legacy_mean_crps": 0.879366, + "emos_mean_crps": 0.989336, + "legacy_mean_mae": 1.022, + "emos_mean_mae": 1.450474, + "legacy_bucket_hit_rate": 0.2, + "emos_bucket_hit_rate": 0.2 + }, + "toronto": { + "samples": 2, + "legacy_mean_crps": 5.497916, + "emos_mean_crps": 6.296632, + "legacy_mean_mae": 6.33, + "emos_mean_mae": 7.003984, + "legacy_bucket_hit_rate": 0.0, + "emos_bucket_hit_rate": 0.0 + }, + "warsaw": { + "samples": 3, + "legacy_mean_crps": 1.618875, + "emos_mean_crps": 1.649515, + "legacy_mean_mae": 2.056667, + "emos_mean_mae": 2.073494, + "legacy_bucket_hit_rate": 0.333333, + "emos_bucket_hit_rate": 0.0 + }, + "wellington": { + "samples": 2, + "legacy_mean_crps": 0.364919, + "emos_mean_crps": 0.939463, + "legacy_mean_mae": 0.15, + "emos_mean_mae": 1.007263, + "legacy_bucket_hit_rate": 1.0, + "emos_bucket_hit_rate": 0.5 } } } \ No newline at end of file diff --git a/scripts/evaluate_probability_calibration.py b/scripts/evaluate_probability_calibration.py index 7807733d..59f7439f 100644 --- a/scripts/evaluate_probability_calibration.py +++ b/scripts/evaluate_probability_calibration.py @@ -18,9 +18,14 @@ from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402 from scripts.fit_probability_calibration import ( # noqa: E402 _default_history_arg, _extract_samples, + _load_history_with_fallback, _load_json_if_exists, + _load_legacy_training_samples, + _load_snapshot_rows, + _load_training_feature_history, + _load_truth_history, + merge_samples_with_legacy_archive, ) -from src.analysis.deb_algorithm import load_history # noqa: E402 def _mean(values): @@ -93,12 +98,20 @@ def main(): ) args = parser.parse_args() - history = load_history(args.history_file) + history = _load_history_with_fallback(args.history_file) + training_feature_history = _load_training_feature_history() + truth_history = _load_truth_history() settlement_history = _load_json_if_exists(args.settlement_history) + snapshot_rows = _load_snapshot_rows(None) + legacy_training_samples = _load_legacy_training_samples() samples, filled_actual_from_history = _extract_samples( history, + training_feature_history=training_feature_history, + truth_history=truth_history, settlement_history=settlement_history, + snapshot_rows=snapshot_rows, ) + samples = merge_samples_with_legacy_archive(samples, legacy_training_samples) legacy_crps = [] emos_crps = [] diff --git a/scripts/fit_probability_calibration.py b/scripts/fit_probability_calibration.py index 31518ce0..18e15d13 100644 --- a/scripts/fit_probability_calibration.py +++ b/scripts/fit_probability_calibration.py @@ -42,6 +42,15 @@ def _load_json_if_exists(path): return data if isinstance(data, dict) else {} +def _legacy_training_samples_path(): + return os.path.join( + PROJECT_ROOT, + "artifacts", + "probability_calibration", + "training_samples.json", + ) + + def _legacy_history_path(): return os.path.join(PROJECT_ROOT, "data", "daily_records.json") @@ -107,6 +116,14 @@ def _load_snapshot_rows(path): return rows +def _load_legacy_training_samples(path=None): + payload = _load_json_if_exists(path or _legacy_training_samples_path()) + rows = payload.get("samples") if isinstance(payload, dict) else None + if not isinstance(rows, list): + return [] + return [row for row in rows if isinstance(row, dict)] + + def _actual_high_for(history, truth_history, settlement_history, city, date_str): city_rows = (history or {}).get(city) or {} record = city_rows.get(date_str) or {} @@ -327,6 +344,40 @@ def _extract_samples(history, training_feature_history=None, truth_history=None, return snapshot_samples + daily_samples, snapshot_filled + daily_filled +def merge_samples_with_legacy_archive(samples, legacy_samples=None): + merged = [] + seen = set() + for sample in samples or []: + if not isinstance(sample, dict): + continue + key = ( + str(sample.get("city") or "").strip().lower(), + str(sample.get("date") or "").strip(), + str(sample.get("sample_source") or "").strip().lower(), + ) + if not key[0] or not key[1]: + continue + if key in seen: + continue + merged.append(sample) + seen.add(key) + for sample in legacy_samples or []: + if not isinstance(sample, dict): + continue + key = ( + str(sample.get("city") or "").strip().lower(), + str(sample.get("date") or "").strip(), + str(sample.get("sample_source") or "").strip().lower(), + ) + if not key[0] or not key[1]: + continue + if key in seen: + continue + merged.append(sample) + seen.add(key) + return merged + + def main(): parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.") parser.add_argument( @@ -366,6 +417,7 @@ def main(): truth_history = _load_truth_history() settlement_history = _load_json_if_exists(args.settlement_history) snapshot_rows = _load_snapshot_rows(args.snapshot_file) + legacy_training_samples = _load_legacy_training_samples() samples, filled_actual_from_history = _extract_samples( history, training_feature_history=training_feature_history, @@ -373,6 +425,7 @@ def main(): settlement_history=settlement_history, snapshot_rows=snapshot_rows, ) + samples = merge_samples_with_legacy_archive(samples, legacy_training_samples) calibration = fit_calibration(samples, version=args.version) if not samples: calibration = default_calibration_payload( @@ -382,6 +435,7 @@ def main(): calibration.setdefault("metrics", {}) calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history calibration["metrics"]["settlement_history_city_count"] = len(settlement_history) + calibration["metrics"]["legacy_archive_samples"] = len(legacy_training_samples) try: calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT) except ValueError: diff --git a/src/analysis/deb_algorithm.py b/src/analysis/deb_algorithm.py index 5faf7555..cc801bf7 100644 --- a/src/analysis/deb_algorithm.py +++ b/src/analysis/deb_algorithm.py @@ -882,8 +882,8 @@ def update_daily_record( except Exception as e: logger.error(f"Error persisting training feature record city={city_name} date={date_str}: {e}") - # 自动清理:只保留最近 14 天的记录(DEB 只用 7 天,14 天留足余量) - cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d") + # 自动清理:训练特征需要更长窗口,保留最近 180 天的记录 + cutoff = (datetime.now() - timedelta(days=180)).strftime("%Y-%m-%d") for city in list(data.keys()): old_dates = [d for d in data[city] if d < cutoff] for d in old_dates: @@ -892,7 +892,7 @@ def update_daily_record( if mode == STATE_STORAGE_SQLITE: try: _daily_record_repo.upsert_record(city_name, date_str, existing) - cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d") + cutoff = (datetime.now() - timedelta(days=180)).strftime("%Y-%m-%d") _daily_record_repo.delete_older_than(cutoff) except Exception as e: logger.error(f"Error upserting daily record to sqlite city={city_name} date={date_str}: {e}")