Guard DEB bucket calibration
This commit is contained in:
@@ -5,11 +5,13 @@ from pathlib import Path
|
||||
|
||||
from src.analysis.deb_evaluation import (
|
||||
DEB_BUCKET_CALIBRATED_VERSION,
|
||||
DEB_GUARDED_CALIBRATED_VERSION,
|
||||
DEB_RAW_VERSION,
|
||||
DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||
backtest_deb_versions,
|
||||
build_bucket_calibrated_corrector,
|
||||
build_recent_bias_corrector,
|
||||
choose_guarded_deb_correction,
|
||||
evaluate_prediction_records,
|
||||
write_backtest_report,
|
||||
)
|
||||
@@ -69,6 +71,58 @@ def test_bucket_calibrated_corrector_optimizes_settlement_bucket_hits():
|
||||
assert corrected["samples"] == 5
|
||||
|
||||
|
||||
def test_guarded_deb_correction_rejects_bucket_when_recent_holdout_gets_worse():
|
||||
history = [
|
||||
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-21", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-22", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-23", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-24", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-25", "deb_prediction": 20.49, "actual_high": 20.49},
|
||||
{"city": "ankara", "target_date": "2026-05-26", "deb_prediction": 20.49, "actual_high": 20.49},
|
||||
{"city": "ankara", "target_date": "2026-05-27", "deb_prediction": 20.49, "actual_high": 20.49},
|
||||
]
|
||||
|
||||
corrected = choose_guarded_deb_correction(
|
||||
history,
|
||||
"ankara",
|
||||
raw_prediction=20.49,
|
||||
min_samples=3,
|
||||
validation_samples=3,
|
||||
)
|
||||
|
||||
assert corrected["version"] == DEB_GUARDED_CALIBRATED_VERSION
|
||||
assert corrected["selected_version"] == DEB_RECENT_BIAS_CORRECTED_VERSION
|
||||
assert corrected["corrected_prediction"] == 20.5
|
||||
assert corrected["guard_reason"] == "bucket_rejected_holdout"
|
||||
|
||||
|
||||
def test_guarded_deb_correction_accepts_bucket_when_holdout_improves():
|
||||
history = [
|
||||
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-21", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-22", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-23", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-24", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-25", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-26", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-27", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
]
|
||||
|
||||
corrected = choose_guarded_deb_correction(
|
||||
history,
|
||||
"ankara",
|
||||
raw_prediction=20.49,
|
||||
min_samples=3,
|
||||
validation_samples=3,
|
||||
)
|
||||
|
||||
assert corrected["version"] == DEB_GUARDED_CALIBRATED_VERSION
|
||||
assert corrected["selected_version"] == DEB_BUCKET_CALIBRATED_VERSION
|
||||
assert corrected["corrected_prediction"] == 20.6
|
||||
assert corrected["guard_reason"] == "bucket_selected_holdout"
|
||||
|
||||
|
||||
def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions():
|
||||
history = [
|
||||
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.0, "actual_high": 22.0},
|
||||
@@ -83,6 +137,7 @@ def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions():
|
||||
assert report["versions"][DEB_RAW_VERSION]["samples"] == 2
|
||||
assert report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["samples"] == 2
|
||||
assert report["versions"][DEB_BUCKET_CALIBRATED_VERSION]["samples"] == 0
|
||||
assert report["versions"][DEB_GUARDED_CALIBRATED_VERSION]["samples"] == 2
|
||||
assert (
|
||||
report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["mae"]
|
||||
< report["versions"][DEB_RAW_VERSION]["mae"]
|
||||
|
||||
@@ -123,7 +123,9 @@ def test_calculate_deb_prediction_keeps_raw_and_adds_versioned_bias_correction(m
|
||||
|
||||
assert result["raw_prediction"] == 24.0
|
||||
assert result["prediction"] == 25.0
|
||||
assert result["version"] == "deb_v1_recent_bias_corrected"
|
||||
assert result["version"] == "deb_v3_guarded_calibrated"
|
||||
assert result["selected_version"] == "deb_v1_recent_bias_corrected"
|
||||
assert result["guard_reason"] == "bucket_unavailable"
|
||||
assert result["bias_adjustment"] == 1.0
|
||||
assert result["bias_samples"] == 3
|
||||
|
||||
@@ -169,7 +171,9 @@ def test_calculate_deb_prediction_prefers_bucket_calibration_when_enough_samples
|
||||
|
||||
assert result["raw_prediction"] == 25.4
|
||||
assert result["prediction"] == 26.0
|
||||
assert result["version"] == "deb_v2_bucket_calibrated"
|
||||
assert result["version"] == "deb_v3_guarded_calibrated"
|
||||
assert result["selected_version"] == "deb_v2_bucket_calibrated"
|
||||
assert result["guard_reason"] == "bucket_same_adjustment"
|
||||
assert result["bias_adjustment"] == 0.6
|
||||
assert result["bias_samples"] == 5
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ def test_training_accuracy_payload_includes_recent_deb_summary():
|
||||
assert payload["deb_summary"]["recent_14d"]["samples"] == 5
|
||||
assert "deb_v1_raw" in payload["deb_summary"]["versions"]
|
||||
assert "deb_v2_bucket_calibrated" in payload["deb_summary"]["versions"]
|
||||
assert "deb_v3_guarded_calibrated" in payload["deb_summary"]["versions"]
|
||||
|
||||
|
||||
def test_training_accuracy_payload_includes_city_deb_trust_strategy():
|
||||
|
||||
Reference in New Issue
Block a user