gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand calibration samples and extend training retention

Browse Source
This commit is contained in:
2569718930@qq.com
2026-04-16 01:05:43 +08:00
parent 53d2ae4c10
commit 611e2d9009
5 changed files with 384 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
artifacts/probability_calibration/default.json
+69 -44
View File
@@ -1,20 +1,20 @@
{
"version": "emos-20260415164136",
"trained_at": "2026-04-15T16:41:36.704829+00:00",
"version": "emos-20260415170349",
"trained_at": "2026-04-15T17:03:49.122865+00:00",
"global": {
"mu": {
"intercept": -0.39094186,
"raw_mu_coef": 0.27476248,
"deb_coef": 0.91159557,
"ens_median_coef": -0.16959018,
"max_so_far_gap_coef": -0.94232523
"intercept": 2.45853035,
"raw_mu_coef": 2.03689031,
"deb_coef": -1.41207502,
"ens_median_coef": 0.08616811,
"max_so_far_gap_coef": -0.5886898
},
"sigma": {
"intercept": -1.56485068,
"raw_sigma_coef": 0.12930442,
"spread_coef": 0.15947819,
"peak_flag_coef": 0.19328621,
"max_so_far_gap_coef": -0.08469752
"intercept": 0.61260118,
"raw_sigma_coef": -1.48747774,
"spread_coef": 2.24562634,
"peak_flag_coef": -0.15906747,
"max_so_far_gap_coef": -0.01835251
}
},
"sigma_constraints": {
@@ -29,56 +29,81 @@
"max_bucket_brier_increase": 0.05
},
"blending": {
"alpha_mu": 1.0,
"alpha_sigma": 0.5
"alpha_mu": 0.05,
"alpha_sigma": 1.0
},
"cities": {
"tokyo": {
"samples": 4,
"mu_bias": 0.211244,
"sigma_scale": 0.5,
"confidence": 0.5
"samples": 5,
"mu_bias": 2.223946,
"sigma_scale": 2.0,
"confidence": 0.625
},
"ankara": {
"samples": 11,
"mu_bias": 0.027471,
"sigma_scale": 0.5,
"confidence": 1.0
"samples": 6,
"mu_bias": -0.396953,
"sigma_scale": 1.094474,
"confidence": 0.75
},
"istanbul": {
"samples": 12,
"mu_bias": 0.048913,
"samples": 3,
"mu_bias": -0.163153,
"sigma_scale": 0.5,
"confidence": 1.0
"confidence": 0.375
},
"hong kong": {
"samples": 12,
"mu_bias": -0.026823,
"sigma_scale": 1.020494,
"confidence": 1.0
"samples": 7,
"mu_bias": 3.267553,
"sigma_scale": 2.0,
"confidence": 0.875
},
"taipei": {
"samples": 5,
"mu_bias": 0.192719,
"sigma_scale": 0.5,
"mu_bias": 4.80044,
"sigma_scale": 2.0,
"confidence": 0.625
},
"seoul": {
"samples": 3,
"mu_bias": -3.080962,
"sigma_scale": 1.437353,
"confidence": 0.375
},
"milan": {
"samples": 3,
"mu_bias": -4.20255,
"sigma_scale": 2.0,
"confidence": 0.375
},
"shanghai": {
"samples": 3,
"mu_bias": 1.185335,
"sigma_scale": 1.68386,
"confidence": 0.375
},
"warsaw": {
"samples": 3,
"mu_bias": -1.574246,
"sigma_scale": 1.343402,
"confidence": 0.375
}
},
"metrics": {
"sample_count": 50,
"mean_crps": 0.278798,
"legacy_mean_crps": 1.26525,
"legacy_mean_mae": 1.4674,
"legacy_bucket_hit_rate": 0.54,
"legacy_bucket_brier": 0.744518,
"selected_mean_crps": 0.278798,
"selected_mean_mae": 0.248457,
"selected_bucket_hit_rate": 0.82,
"selected_bucket_brier": 0.369542,
"selected_score": 0.8508,
"legacy_score": 2.660378,
"sample_count": 71,
"mean_crps": 3.226613,
"legacy_mean_crps": 3.608885,
"legacy_mean_mae": 3.81507,
"legacy_bucket_hit_rate": 0.492958,
"legacy_bucket_brier": 0.873776,
"selected_mean_crps": 3.44742,
"selected_mean_mae": 3.749078,
"selected_bucket_hit_rate": 0.492958,
"selected_bucket_brier": 0.878621,
"selected_score": 5.241857,
"legacy_score": 5.406287,
"filled_actual_from_history": 0,
"settlement_history_city_count": 30
"settlement_history_city_count": 30,
"legacy_archive_samples": 54
},
"source": "artifacts\\probability_calibration\\default.json"
}
artifacts/probability_calibration/evaluation_report.json
+243 -54
View File
@@ -1,95 +1,284 @@
{
"summary": {
"sample_count": 17,
"sample_count": 71,
"filled_actual_from_history": 0,
"legacy": {
"mean_crps": 1.095786,
"mean_mae": 1.243529,
"bucket_hit_rate": 0.588235
"mean_crps": 3.608885,
"mean_mae": 3.81507,
"bucket_hit_rate": 0.492958
},
"emos": {
"mean_crps": 1.257578,
"mean_mae": 1.545328,
"bucket_hit_rate": 0.352941
"mean_crps": 3.881798,
"mean_mae": 4.378282,
"bucket_hit_rate": 0.225352
},
"delta": {
"crps": 0.161791,
"mae": 0.301798,
"bucket_hit_rate": -0.235294
"crps": 0.272913,
"mae": 0.563211,
"bucket_hit_rate": -0.267606
}
},
"by_city": {
"ankara": {
"samples": 3,
"legacy_mean_crps": 2.707176,
"emos_mean_crps": 2.632737,
"legacy_mean_mae": 3.0,
"emos_mean_mae": 2.90439,
"legacy_bucket_hit_rate": 0.333333,
"samples": 6,
"legacy_mean_crps": 1.365135,
"emos_mean_crps": 1.454394,
"legacy_mean_mae": 1.466667,
"emos_mean_mae": 1.781865,
"legacy_bucket_hit_rate": 0.666667,
"emos_bucket_hit_rate": 0.333333
},
"atlanta": {
"samples": 2,
"legacy_mean_crps": 30.449382,
"emos_mean_crps": 30.2136,
"legacy_mean_mae": 32.015,
"emos_mean_mae": 31.431544,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"buenos aires": {
"samples": 2,
"legacy_mean_crps": 9.113412,
"emos_mean_crps": 9.25091,
"legacy_mean_mae": 10.27,
"emos_mean_mae": 10.234011,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"busan": {
"samples": 1,
"legacy_mean_crps": 0.264698,
"emos_mean_crps": 0.797946,
"legacy_mean_crps": 0.451701,
"emos_mean_crps": 1.176676,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 1.213185,
"emos_mean_mae": 1.887548,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"chengdu": {
"samples": 1,
"legacy_mean_crps": 0.218827,
"emos_mean_crps": 0.321502,
"legacy_mean_crps": 0.306469,
"emos_mean_crps": 0.555327,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.539971,
"emos_mean_mae": 0.929502,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"chicago": {
"samples": 1,
"legacy_mean_crps": 1.250268,
"emos_mean_crps": 2.090175,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 3.396557,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"dallas": {
"samples": 1,
"legacy_mean_crps": 2.173363,
"emos_mean_crps": 1.05572,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 1.653475,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"hong kong": {
"samples": 3,
"legacy_mean_crps": 0.321946,
"emos_mean_crps": 0.419198,
"legacy_mean_mae": 0.27,
"emos_mean_mae": 0.673605,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
"samples": 7,
"legacy_mean_crps": 0.706099,
"emos_mean_crps": 1.573367,
"legacy_mean_mae": 0.657143,
"emos_mean_mae": 1.88803,
"legacy_bucket_hit_rate": 0.714286,
"emos_bucket_hit_rate": 0.142857
},
"istanbul": {
"samples": 3,
"legacy_mean_crps": 1.308328,
"emos_mean_crps": 1.231189,
"legacy_mean_mae": 1.38,
"emos_mean_mae": 1.211693,
"legacy_bucket_hit_rate": 0.666667,
"legacy_mean_crps": 1.264103,
"emos_mean_crps": 1.266844,
"legacy_mean_mae": 1.5,
"emos_mean_mae": 1.415758,
"legacy_bucket_hit_rate": 0.333333,
"emos_bucket_hit_rate": 0.666667
},
"seoul": {
"samples": 1,
"legacy_mean_crps": 0.208763,
"emos_mean_crps": 2.708973,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 3.02868,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"taipei": {
"london": {
"samples": 2,
"legacy_mean_crps": 0.767971,
"emos_mean_crps": 1.205469,
"legacy_mean_mae": 1.195,
"emos_mean_mae": 1.937374,
"legacy_mean_crps": 3.885033,
"emos_mean_crps": 4.057521,
"legacy_mean_mae": 4.135,
"emos_mean_mae": 4.313103,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"tokyo": {
"lucknow": {
"samples": 2,
"legacy_mean_crps": 2.487193,
"emos_mean_crps": 2.021264,
"legacy_mean_mae": 3.205,
"emos_mean_mae": 2.47462,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.5
},
"madrid": {
"samples": 2,
"legacy_mean_crps": 6.27726,
"emos_mean_crps": 6.588692,
"legacy_mean_mae": 7.33,
"emos_mean_mae": 7.486713,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"miami": {
"samples": 2,
"legacy_mean_crps": 28.637631,
"emos_mean_crps": 28.800796,
"legacy_mean_mae": 30.175,
"emos_mean_mae": 30.107593,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"milan": {
"samples": 3,
"legacy_mean_crps": 1.129263,
"emos_mean_crps": 0.763363,
"legacy_mean_mae": 1.3,
"emos_mean_mae": 1.081641,
"legacy_bucket_hit_rate": 0.333333,
"legacy_mean_crps": 4.401392,
"emos_mean_crps": 4.947416,
"legacy_mean_mae": 4.06,
"emos_mean_mae": 5.6958,
"legacy_bucket_hit_rate": 0.666667,
"emos_bucket_hit_rate": 0.333333
},
"munich": {
"samples": 2,
"legacy_mean_crps": 3.145192,
"emos_mean_crps": 3.200946,
"legacy_mean_mae": 3.64,
"emos_mean_mae": 3.634475,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"new york": {
"samples": 1,
"legacy_mean_crps": 3.692845,
"emos_mean_crps": 3.881457,
"legacy_mean_mae": 4.94,
"emos_mean_mae": 4.954702,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"paris": {
"samples": 2,
"legacy_mean_crps": 4.013782,
"emos_mean_crps": 4.130614,
"legacy_mean_mae": 4.265,
"emos_mean_mae": 4.314216,
"legacy_bucket_hit_rate": 0.5,
"emos_bucket_hit_rate": 0.5
},
"sao paulo": {
"samples": 2,
"legacy_mean_crps": 5.540967,
"emos_mean_crps": 5.483258,
"legacy_mean_mae": 6.57,
"emos_mean_mae": 6.358434,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"seattle": {
"samples": 1,
"legacy_mean_crps": 0.315488,
"emos_mean_crps": 0.271998,
"legacy_mean_mae": 0.0,
"emos_mean_mae": 0.105037,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 1.0
},
"seoul": {
"samples": 3,
"legacy_mean_crps": 0.508331,
"emos_mean_crps": 1.718042,
"legacy_mean_mae": 0.2,
"emos_mean_mae": 2.57121,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.0
},
"shanghai": {
"samples": 3,
"legacy_mean_crps": 0.299116,
"emos_mean_crps": 0.553444,
"legacy_mean_mae": 0.1,
"emos_mean_mae": 0.71746,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.666667
},
"shenzhen": {
"samples": 1,
"legacy_mean_crps": 1.198351,
"emos_mean_crps": 0.507942,
"legacy_mean_mae": 1.3,
"emos_mean_mae": 0.644561,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"singapore": {
"samples": 2,
"legacy_mean_crps": 0.281993,
"emos_mean_crps": 0.877023,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 1.142765,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.5
},
"taipei": {
"samples": 5,
"legacy_mean_crps": 0.950739,
"emos_mean_crps": 1.257972,
"legacy_mean_mae": 0.94,
"emos_mean_mae": 1.687014,
"legacy_bucket_hit_rate": 0.6,
"emos_bucket_hit_rate": 0.2
},
"tel aviv": {
"samples": 2,
"legacy_mean_crps": 0.446758,
"emos_mean_crps": 0.482341,
"legacy_mean_mae": 0.3,
"emos_mean_mae": 0.62659,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.5
},
"tokyo": {
"samples": 5,
"legacy_mean_crps": 0.879366,
"emos_mean_crps": 0.989336,
"legacy_mean_mae": 1.022,
"emos_mean_mae": 1.450474,
"legacy_bucket_hit_rate": 0.2,
"emos_bucket_hit_rate": 0.2
},
"toronto": {
"samples": 2,
"legacy_mean_crps": 5.497916,
"emos_mean_crps": 6.296632,
"legacy_mean_mae": 6.33,
"emos_mean_mae": 7.003984,
"legacy_bucket_hit_rate": 0.0,
"emos_bucket_hit_rate": 0.0
},
"warsaw": {
"samples": 3,
"legacy_mean_crps": 1.618875,
"emos_mean_crps": 1.649515,
"legacy_mean_mae": 2.056667,
"emos_mean_mae": 2.073494,
"legacy_bucket_hit_rate": 0.333333,
"emos_bucket_hit_rate": 0.0
},
"wellington": {
"samples": 2,
"legacy_mean_crps": 0.364919,
"emos_mean_crps": 0.939463,
"legacy_mean_mae": 0.15,
"emos_mean_mae": 1.007263,
"legacy_bucket_hit_rate": 1.0,
"emos_bucket_hit_rate": 0.5
}
}
}
scripts/evaluate_probability_calibration.py
+15 -2
View File
@@ -18,9 +18,14 @@ from src.analysis.settlement_rounding import apply_city_settlement # noqa: E402
from scripts.fit_probability_calibration import ( # noqa: E402
_default_history_arg,
_extract_samples,
_load_history_with_fallback,
_load_json_if_exists,
_load_legacy_training_samples,
_load_snapshot_rows,
_load_training_feature_history,
_load_truth_history,
merge_samples_with_legacy_archive,
)
from src.analysis.deb_algorithm import load_history # noqa: E402
def _mean(values):
@@ -93,12 +98,20 @@ def main():
)
args = parser.parse_args()
history = load_history(args.history_file)
history = _load_history_with_fallback(args.history_file)
training_feature_history = _load_training_feature_history()
truth_history = _load_truth_history()
settlement_history = _load_json_if_exists(args.settlement_history)
snapshot_rows = _load_snapshot_rows(None)
legacy_training_samples = _load_legacy_training_samples()
samples, filled_actual_from_history = _extract_samples(
history,
training_feature_history=training_feature_history,
truth_history=truth_history,
settlement_history=settlement_history,
snapshot_rows=snapshot_rows,
)
samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
legacy_crps = []
emos_crps = []
scripts/fit_probability_calibration.py
+54
View File
@@ -42,6 +42,15 @@ def _load_json_if_exists(path):
return data if isinstance(data, dict) else {}
def _legacy_training_samples_path():
return os.path.join(
PROJECT_ROOT,
"artifacts",
"probability_calibration",
"training_samples.json",
)
def _legacy_history_path():
return os.path.join(PROJECT_ROOT, "data", "daily_records.json")
@@ -107,6 +116,14 @@ def _load_snapshot_rows(path):
return rows
def _load_legacy_training_samples(path=None):
payload = _load_json_if_exists(path or _legacy_training_samples_path())
rows = payload.get("samples") if isinstance(payload, dict) else None
if not isinstance(rows, list):
return []
return [row for row in rows if isinstance(row, dict)]
def _actual_high_for(history, truth_history, settlement_history, city, date_str):
city_rows = (history or {}).get(city) or {}
record = city_rows.get(date_str) or {}
@@ -327,6 +344,40 @@ def _extract_samples(history, training_feature_history=None, truth_history=None,
return snapshot_samples + daily_samples, snapshot_filled + daily_filled
def merge_samples_with_legacy_archive(samples, legacy_samples=None):
merged = []
seen = set()
for sample in samples or []:
if not isinstance(sample, dict):
continue
key = (
str(sample.get("city") or "").strip().lower(),
str(sample.get("date") or "").strip(),
str(sample.get("sample_source") or "").strip().lower(),
)
if not key[0] or not key[1]:
continue
if key in seen:
continue
merged.append(sample)
seen.add(key)
for sample in legacy_samples or []:
if not isinstance(sample, dict):
continue
key = (
str(sample.get("city") or "").strip().lower(),
str(sample.get("date") or "").strip(),
str(sample.get("sample_source") or "").strip().lower(),
)
if not key[0] or not key[1]:
continue
if key in seen:
continue
merged.append(sample)
seen.add(key)
return merged
def main():
parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
parser.add_argument(
@@ -366,6 +417,7 @@ def main():
truth_history = _load_truth_history()
settlement_history = _load_json_if_exists(args.settlement_history)
snapshot_rows = _load_snapshot_rows(args.snapshot_file)
legacy_training_samples = _load_legacy_training_samples()
samples, filled_actual_from_history = _extract_samples(
history,
training_feature_history=training_feature_history,
@@ -373,6 +425,7 @@ def main():
settlement_history=settlement_history,
snapshot_rows=snapshot_rows,
)
samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
calibration = fit_calibration(samples, version=args.version)
if not samples:
calibration = default_calibration_payload(
@@ -382,6 +435,7 @@ def main():
calibration.setdefault("metrics", {})
calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history
calibration["metrics"]["settlement_history_city_count"] = len(settlement_history)
calibration["metrics"]["legacy_archive_samples"] = len(legacy_training_samples)
try:
calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT)
except ValueError:
src/analysis/deb_algorithm.py
+3 -3
View File
@@ -882,8 +882,8 @@ def update_daily_record(
except Exception as e:
logger.error(f"Error persisting training feature record city={city_name} date={date_str}: {e}")
# 自动清理：只保留最近 14 天的记录（DEB 只用 7 天，14 天留足余量）
cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d")
# 自动清理：训练特征需要更长窗口，保留最近 180 天的记录
cutoff = (datetime.now() - timedelta(days=180)).strftime("%Y-%m-%d")
for city in list(data.keys()):
old_dates = [d for d in data[city] if d < cutoff]
for d in old_dates:
@@ -892,7 +892,7 @@ def update_daily_record(
if mode == STATE_STORAGE_SQLITE:
try:
_daily_record_repo.upsert_record(city_name, date_str, existing)
cutoff = (datetime.now() - timedelta(days=14)).strftime("%Y-%m-%d")
cutoff = (datetime.now() - timedelta(days=180)).strftime("%Y-%m-%d")
_daily_record_repo.delete_older_than(cutoff)
except Exception as e:
logger.error(f"Error upserting daily record to sqlite city={city_name} date={date_str}: {e}")