Expand calibration samples and extend training retention

scripts/evaluate_probability_calibration.py

@@ -18,9 +18,14 @@ from src.analysis.settlement_rounding import apply_city_settlement  # noqa: E402
 from scripts.fit_probability_calibration import (  # noqa: E402
     _default_history_arg,
     _extract_samples,
+    _load_history_with_fallback,
     _load_json_if_exists,
+    _load_legacy_training_samples,
+    _load_snapshot_rows,
+    _load_training_feature_history,
+    _load_truth_history,
+    merge_samples_with_legacy_archive,
 )
-from src.analysis.deb_algorithm import load_history  # noqa: E402
 
 
 def _mean(values):
@@ -93,12 +98,20 @@ def main():
     )
     args = parser.parse_args()
 
-    history = load_history(args.history_file)
+    history = _load_history_with_fallback(args.history_file)
+    training_feature_history = _load_training_feature_history()
+    truth_history = _load_truth_history()
     settlement_history = _load_json_if_exists(args.settlement_history)
+    snapshot_rows = _load_snapshot_rows(None)
+    legacy_training_samples = _load_legacy_training_samples()
     samples, filled_actual_from_history = _extract_samples(
         history,
+        training_feature_history=training_feature_history,
+        truth_history=truth_history,
         settlement_history=settlement_history,
+        snapshot_rows=snapshot_rows,
     )
+    samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
 
     legacy_crps = []
     emos_crps = []

scripts/fit_probability_calibration.py

@@ -42,6 +42,15 @@ def _load_json_if_exists(path):
     return data if isinstance(data, dict) else {}
 
 
+def _legacy_training_samples_path():
+    return os.path.join(
+        PROJECT_ROOT,
+        "artifacts",
+        "probability_calibration",
+        "training_samples.json",
+    )
+
+
 def _legacy_history_path():
     return os.path.join(PROJECT_ROOT, "data", "daily_records.json")
 
@@ -107,6 +116,14 @@ def _load_snapshot_rows(path):
     return rows
 
 
+def _load_legacy_training_samples(path=None):
+    payload = _load_json_if_exists(path or _legacy_training_samples_path())
+    rows = payload.get("samples") if isinstance(payload, dict) else None
+    if not isinstance(rows, list):
+        return []
+    return [row for row in rows if isinstance(row, dict)]
+
+
 def _actual_high_for(history, truth_history, settlement_history, city, date_str):
     city_rows = (history or {}).get(city) or {}
     record = city_rows.get(date_str) or {}
@@ -327,6 +344,40 @@ def _extract_samples(history, training_feature_history=None, truth_history=None,
     return snapshot_samples + daily_samples, snapshot_filled + daily_filled
 
 
+def merge_samples_with_legacy_archive(samples, legacy_samples=None):
+    merged = []
+    seen = set()
+    for sample in samples or []:
+        if not isinstance(sample, dict):
+            continue
+        key = (
+            str(sample.get("city") or "").strip().lower(),
+            str(sample.get("date") or "").strip(),
+            str(sample.get("sample_source") or "").strip().lower(),
+        )
+        if not key[0] or not key[1]:
+            continue
+        if key in seen:
+            continue
+        merged.append(sample)
+        seen.add(key)
+    for sample in legacy_samples or []:
+        if not isinstance(sample, dict):
+            continue
+        key = (
+            str(sample.get("city") or "").strip().lower(),
+            str(sample.get("date") or "").strip(),
+            str(sample.get("sample_source") or "").strip().lower(),
+        )
+        if not key[0] or not key[1]:
+            continue
+        if key in seen:
+            continue
+        merged.append(sample)
+        seen.add(key)
+    return merged
+
+
 def main():
     parser = argparse.ArgumentParser(description="Fit PolyWeather probability calibration parameters.")
     parser.add_argument(
@@ -366,6 +417,7 @@ def main():
     truth_history = _load_truth_history()
     settlement_history = _load_json_if_exists(args.settlement_history)
     snapshot_rows = _load_snapshot_rows(args.snapshot_file)
+    legacy_training_samples = _load_legacy_training_samples()
     samples, filled_actual_from_history = _extract_samples(
         history,
         training_feature_history=training_feature_history,
@@ -373,6 +425,7 @@ def main():
         settlement_history=settlement_history,
         snapshot_rows=snapshot_rows,
     )
+    samples = merge_samples_with_legacy_archive(samples, legacy_training_samples)
     calibration = fit_calibration(samples, version=args.version)
     if not samples:
         calibration = default_calibration_payload(
@@ -382,6 +435,7 @@ def main():
     calibration.setdefault("metrics", {})
     calibration["metrics"]["filled_actual_from_history"] = filled_actual_from_history
     calibration["metrics"]["settlement_history_city_count"] = len(settlement_history)
+    calibration["metrics"]["legacy_archive_samples"] = len(legacy_training_samples)
     try:
         calibration["source"] = os.path.relpath(args.output, PROJECT_ROOT)
     except ValueError: