Guard DEB bucket calibration
This commit is contained in:
@@ -212,6 +212,7 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
||||
{ key: "deb_v1_raw", label: isEn ? "Raw DEB" : "原始 DEB" },
|
||||
{ key: "deb_v1_recent_bias_corrected", label: isEn ? "Mean Bias" : "均值偏差" },
|
||||
{ key: "deb_v2_bucket_calibrated", label: isEn ? "Bucket v2" : "桶校准 v2" },
|
||||
{ key: "deb_v3_guarded_calibrated", label: isEn ? "Guarded v3" : "保护 v3" },
|
||||
].map(({ key, label }) => ({ key, label, value: versions[key] })).filter((row) => row.value);
|
||||
}, [debSummary?.versions, isEn]);
|
||||
|
||||
|
||||
@@ -225,6 +225,8 @@ export interface DebForecast {
|
||||
prediction: number | null;
|
||||
raw_prediction?: number | null;
|
||||
version?: string | null;
|
||||
selected_version?: string | null;
|
||||
guard_reason?: string | null;
|
||||
weights_info?: string | null;
|
||||
bias_adjustment?: number | null;
|
||||
bias_samples?: number | null;
|
||||
|
||||
@@ -1341,10 +1341,10 @@ def calculate_deb_prediction(
|
||||
"""
|
||||
from src.analysis.deb_evaluation import (
|
||||
DEB_BUCKET_CALIBRATED_VERSION,
|
||||
DEB_GUARDED_CALIBRATED_VERSION,
|
||||
DEB_RAW_VERSION,
|
||||
DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||
build_bucket_calibrated_corrector,
|
||||
build_recent_bias_corrector,
|
||||
choose_guarded_deb_correction,
|
||||
flatten_daily_records,
|
||||
)
|
||||
|
||||
@@ -1376,18 +1376,14 @@ def calculate_deb_prediction(
|
||||
|
||||
data = load_history(_get_history_file_path())
|
||||
history_rows = flatten_daily_records(data)
|
||||
corrected = build_recent_bias_corrector(
|
||||
corrected = choose_guarded_deb_correction(
|
||||
history_rows,
|
||||
city_name,
|
||||
raw_prediction,
|
||||
lookback_days=bias_lookback_days,
|
||||
min_samples=bias_min_samples,
|
||||
).apply(city_name, raw_prediction)
|
||||
bucket_corrected = build_bucket_calibrated_corrector(
|
||||
history_rows,
|
||||
lookback_days=bias_lookback_days,
|
||||
min_samples=max(5, int(bias_min_samples or 0)),
|
||||
).apply(city_name, raw_prediction)
|
||||
if int(bucket_corrected.get("samples") or 0) > 0:
|
||||
corrected = bucket_corrected
|
||||
bucket_min_samples=max(5, int(bias_min_samples or 0)),
|
||||
)
|
||||
|
||||
bias_adjustment = float(corrected.get("bias_adjustment") or 0.0)
|
||||
bias_samples = int(corrected.get("samples") or 0)
|
||||
@@ -1410,9 +1406,12 @@ def calculate_deb_prediction(
|
||||
|
||||
next_weights_info = weights_info
|
||||
if abs(bias_adjustment) >= 0.05:
|
||||
selected_version = corrected.get("selected_version") or corrected.get("version")
|
||||
correction_label = (
|
||||
"bucket_calibration"
|
||||
if corrected.get("version") == DEB_BUCKET_CALIBRATED_VERSION
|
||||
if selected_version == DEB_BUCKET_CALIBRATED_VERSION
|
||||
else "guarded_bias"
|
||||
if corrected.get("version") == DEB_GUARDED_CALIBRATED_VERSION
|
||||
else "recent_bias"
|
||||
)
|
||||
next_weights_info = (
|
||||
@@ -1427,6 +1426,8 @@ def calculate_deb_prediction(
|
||||
"weights_info": next_weights_info,
|
||||
"bias_adjustment": bias_adjustment,
|
||||
"bias_samples": bias_samples,
|
||||
"selected_version": corrected.get("selected_version"),
|
||||
"guard_reason": corrected.get("guard_reason"),
|
||||
**quality,
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from src.analysis.settlement_rounding import apply_city_settlement
|
||||
DEB_RAW_VERSION = "deb_v1_raw"
|
||||
DEB_RECENT_BIAS_CORRECTED_VERSION = "deb_v1_recent_bias_corrected"
|
||||
DEB_BUCKET_CALIBRATED_VERSION = "deb_v2_bucket_calibrated"
|
||||
DEB_GUARDED_CALIBRATED_VERSION = "deb_v3_guarded_calibrated"
|
||||
DEB_BACKTEST_SCHEMA_VERSION = "deb_backtest_report.v1"
|
||||
|
||||
|
||||
@@ -231,6 +232,207 @@ def build_bucket_calibrated_corrector(
|
||||
)
|
||||
|
||||
|
||||
def _evaluate_city_adjustment(
|
||||
history: Iterable[dict[str, Any]],
|
||||
city: str,
|
||||
*,
|
||||
adjustment: float,
|
||||
lookback_days: int = 30,
|
||||
) -> dict[str, Any]:
|
||||
city_key = str(city or "").strip().lower()
|
||||
rows = [
|
||||
row
|
||||
for record in history
|
||||
if (row := _normalise_record(record)) and row["city"] == city_key
|
||||
]
|
||||
rows.sort(key=lambda row: row["target_date"], reverse=True)
|
||||
recent = rows[: max(int(lookback_days or 0), 1)]
|
||||
hits = 0
|
||||
total = 0
|
||||
errors: list[float] = []
|
||||
for row in recent:
|
||||
prediction = row["prediction"] + float(adjustment or 0.0)
|
||||
actual = row["actual"]
|
||||
try:
|
||||
pred_bucket = apply_city_settlement(city_key, prediction)
|
||||
actual_bucket = apply_city_settlement(city_key, actual)
|
||||
except Exception:
|
||||
continue
|
||||
if pred_bucket is None or actual_bucket is None:
|
||||
continue
|
||||
total += 1
|
||||
if pred_bucket == actual_bucket:
|
||||
hits += 1
|
||||
errors.append(abs(prediction - actual))
|
||||
|
||||
return {
|
||||
"samples": total,
|
||||
"hits": hits,
|
||||
"bucket_hit_rate": _round3(hits / total) if total else None,
|
||||
"mae": _round3(statistics.mean(errors)) if errors else None,
|
||||
}
|
||||
|
||||
|
||||
def _bucket_holdout_is_better(
|
||||
recent_metrics: dict[str, Any],
|
||||
bucket_metrics: dict[str, Any],
|
||||
) -> bool:
|
||||
recent_rate = _sf(recent_metrics.get("bucket_hit_rate"))
|
||||
bucket_rate = _sf(bucket_metrics.get("bucket_hit_rate"))
|
||||
recent_mae = _sf(recent_metrics.get("mae"))
|
||||
bucket_mae = _sf(bucket_metrics.get("mae"))
|
||||
if bucket_rate is None or bucket_mae is None:
|
||||
return False
|
||||
if recent_rate is None or recent_mae is None:
|
||||
return True
|
||||
if bucket_rate > recent_rate and bucket_mae <= recent_mae + 0.25:
|
||||
return True
|
||||
if bucket_rate >= recent_rate and bucket_mae + 0.05 < recent_mae:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _guarded_deb_result(
|
||||
selected: dict[str, Any],
|
||||
*,
|
||||
selected_version: str,
|
||||
guard_reason: str,
|
||||
recent_metrics: dict[str, Any],
|
||||
bucket_metrics: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"version": DEB_GUARDED_CALIBRATED_VERSION,
|
||||
"selected_version": selected_version,
|
||||
"raw_prediction": selected["raw_prediction"],
|
||||
"corrected_prediction": selected["corrected_prediction"],
|
||||
"bias_adjustment": selected["bias_adjustment"],
|
||||
"samples": selected["samples"],
|
||||
"guard_reason": guard_reason,
|
||||
"candidate_metrics": {
|
||||
DEB_RECENT_BIAS_CORRECTED_VERSION: recent_metrics,
|
||||
DEB_BUCKET_CALIBRATED_VERSION: bucket_metrics,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def choose_guarded_deb_correction(
|
||||
history: Iterable[dict[str, Any]],
|
||||
city: str,
|
||||
raw_prediction: float,
|
||||
*,
|
||||
lookback_days: int = 30,
|
||||
min_samples: int = 3,
|
||||
bucket_min_samples: int = 5,
|
||||
validation_samples: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
history_rows = [row for record in history if (row := _normalise_record(record))]
|
||||
city_key = str(city or "").strip().lower()
|
||||
city_rows = [row for row in history_rows if row["city"] == city_key]
|
||||
city_rows.sort(key=lambda row: row["target_date"], reverse=True)
|
||||
recent_rows = city_rows[: max(int(lookback_days or 0), 1)]
|
||||
|
||||
recent = build_recent_bias_corrector(
|
||||
history_rows,
|
||||
lookback_days=lookback_days,
|
||||
min_samples=min_samples,
|
||||
).apply(city_key, raw_prediction)
|
||||
bucket = build_bucket_calibrated_corrector(
|
||||
history_rows,
|
||||
lookback_days=lookback_days,
|
||||
min_samples=bucket_min_samples,
|
||||
).apply(city_key, raw_prediction)
|
||||
|
||||
recent_metrics = _evaluate_city_adjustment(
|
||||
recent_rows,
|
||||
city_key,
|
||||
adjustment=float(recent.get("bias_adjustment") or 0.0),
|
||||
lookback_days=lookback_days,
|
||||
)
|
||||
bucket_metrics = _evaluate_city_adjustment(
|
||||
recent_rows,
|
||||
city_key,
|
||||
adjustment=float(bucket.get("bias_adjustment") or 0.0),
|
||||
lookback_days=lookback_days,
|
||||
)
|
||||
|
||||
if int(bucket.get("samples") or 0) <= 0:
|
||||
return _guarded_deb_result(
|
||||
recent,
|
||||
selected_version=DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||
guard_reason="bucket_unavailable",
|
||||
recent_metrics=recent_metrics,
|
||||
bucket_metrics=bucket_metrics,
|
||||
)
|
||||
|
||||
if abs(float(bucket.get("bias_adjustment") or 0.0) - float(recent.get("bias_adjustment") or 0.0)) < 0.05:
|
||||
return _guarded_deb_result(
|
||||
bucket,
|
||||
selected_version=DEB_BUCKET_CALIBRATED_VERSION,
|
||||
guard_reason="bucket_same_adjustment",
|
||||
recent_metrics=recent_metrics,
|
||||
bucket_metrics=bucket_metrics,
|
||||
)
|
||||
|
||||
safe_validation_samples = max(int(validation_samples or 0), 1)
|
||||
if len(recent_rows) >= int(bucket_min_samples or 0) + safe_validation_samples:
|
||||
validation_rows = recent_rows[:safe_validation_samples]
|
||||
training_rows = recent_rows[safe_validation_samples:]
|
||||
recent_holdout = build_recent_bias_corrector(
|
||||
training_rows,
|
||||
lookback_days=lookback_days,
|
||||
min_samples=min_samples,
|
||||
).apply(city_key, raw_prediction)
|
||||
bucket_holdout = build_bucket_calibrated_corrector(
|
||||
training_rows,
|
||||
lookback_days=lookback_days,
|
||||
min_samples=bucket_min_samples,
|
||||
).apply(city_key, raw_prediction)
|
||||
if int(bucket_holdout.get("samples") or 0) > 0:
|
||||
recent_holdout_metrics = _evaluate_city_adjustment(
|
||||
validation_rows,
|
||||
city_key,
|
||||
adjustment=float(recent_holdout.get("bias_adjustment") or 0.0),
|
||||
lookback_days=safe_validation_samples,
|
||||
)
|
||||
bucket_holdout_metrics = _evaluate_city_adjustment(
|
||||
validation_rows,
|
||||
city_key,
|
||||
adjustment=float(bucket_holdout.get("bias_adjustment") or 0.0),
|
||||
lookback_days=safe_validation_samples,
|
||||
)
|
||||
if _bucket_holdout_is_better(recent_holdout_metrics, bucket_holdout_metrics):
|
||||
return _guarded_deb_result(
|
||||
bucket,
|
||||
selected_version=DEB_BUCKET_CALIBRATED_VERSION,
|
||||
guard_reason="bucket_selected_holdout",
|
||||
recent_metrics=recent_metrics,
|
||||
bucket_metrics=bucket_metrics,
|
||||
)
|
||||
return _guarded_deb_result(
|
||||
recent,
|
||||
selected_version=DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||
guard_reason="bucket_rejected_holdout",
|
||||
recent_metrics=recent_metrics,
|
||||
bucket_metrics=bucket_metrics,
|
||||
)
|
||||
|
||||
if _bucket_holdout_is_better(recent_metrics, bucket_metrics):
|
||||
return _guarded_deb_result(
|
||||
bucket,
|
||||
selected_version=DEB_BUCKET_CALIBRATED_VERSION,
|
||||
guard_reason="bucket_selected_recent",
|
||||
recent_metrics=recent_metrics,
|
||||
bucket_metrics=bucket_metrics,
|
||||
)
|
||||
return _guarded_deb_result(
|
||||
recent,
|
||||
selected_version=DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||
guard_reason="bucket_rejected_recent",
|
||||
recent_metrics=recent_metrics,
|
||||
bucket_metrics=bucket_metrics,
|
||||
)
|
||||
|
||||
|
||||
def backtest_deb_versions(
|
||||
history: Iterable[dict[str, Any]],
|
||||
*,
|
||||
@@ -244,6 +446,7 @@ def backtest_deb_versions(
|
||||
raw_eval_rows: list[dict[str, Any]] = []
|
||||
corrected_eval_rows: list[dict[str, Any]] = []
|
||||
bucket_eval_rows: list[dict[str, Any]] = []
|
||||
guarded_eval_rows: list[dict[str, Any]] = []
|
||||
|
||||
by_city: dict[str, list[dict[str, Any]]] = {}
|
||||
for row in rows:
|
||||
@@ -260,9 +463,17 @@ def backtest_deb_versions(
|
||||
)
|
||||
corrected = corrector.apply(row["city"], row["prediction"])
|
||||
bucket_corrected = bucket_corrector.apply(row["city"], row["prediction"])
|
||||
guarded_corrected = choose_guarded_deb_correction(
|
||||
previous,
|
||||
row["city"],
|
||||
row["prediction"],
|
||||
lookback_days=train_lookback_days,
|
||||
min_samples=min_train_samples,
|
||||
)
|
||||
raw_prediction = round(row["prediction"], 1)
|
||||
corrected_prediction = corrected["corrected_prediction"]
|
||||
bucket_prediction = bucket_corrected["corrected_prediction"]
|
||||
guarded_prediction = guarded_corrected["corrected_prediction"]
|
||||
|
||||
raw_eval_rows.append(
|
||||
{
|
||||
@@ -289,6 +500,14 @@ def backtest_deb_versions(
|
||||
"actual": row["actual"],
|
||||
}
|
||||
)
|
||||
guarded_eval_rows.append(
|
||||
{
|
||||
"city": row["city"],
|
||||
"target_date": row["target_date"],
|
||||
"prediction": guarded_prediction,
|
||||
"actual": row["actual"],
|
||||
}
|
||||
)
|
||||
report_rows.append(
|
||||
{
|
||||
"city": row["city"],
|
||||
@@ -311,6 +530,14 @@ def backtest_deb_versions(
|
||||
"bias_adjustment": bucket_corrected["bias_adjustment"],
|
||||
"train_samples": bucket_corrected["samples"],
|
||||
},
|
||||
DEB_GUARDED_CALIBRATED_VERSION: {
|
||||
"prediction": guarded_prediction,
|
||||
"error": round(guarded_prediction - row["actual"], 3),
|
||||
"bias_adjustment": guarded_corrected["bias_adjustment"],
|
||||
"train_samples": guarded_corrected["samples"],
|
||||
"selected_version": guarded_corrected["selected_version"],
|
||||
"guard_reason": guarded_corrected["guard_reason"],
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -331,6 +558,10 @@ def backtest_deb_versions(
|
||||
bucket_eval_rows,
|
||||
version=DEB_BUCKET_CALIBRATED_VERSION,
|
||||
),
|
||||
DEB_GUARDED_CALIBRATED_VERSION: evaluate_prediction_records(
|
||||
guarded_eval_rows,
|
||||
version=DEB_GUARDED_CALIBRATED_VERSION,
|
||||
),
|
||||
},
|
||||
"rows": report_rows,
|
||||
}
|
||||
@@ -391,6 +622,12 @@ def write_backtest_report(
|
||||
f"{DEB_BUCKET_CALIBRATED_VERSION}_error",
|
||||
f"{DEB_BUCKET_CALIBRATED_VERSION}_bias_adjustment",
|
||||
f"{DEB_BUCKET_CALIBRATED_VERSION}_train_samples",
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_prediction",
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_error",
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_bias_adjustment",
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_train_samples",
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_selected_version",
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_guard_reason",
|
||||
],
|
||||
)
|
||||
writer.writeheader()
|
||||
@@ -399,6 +636,7 @@ def write_backtest_report(
|
||||
raw = versions.get(DEB_RAW_VERSION) or {}
|
||||
corrected = versions.get(DEB_RECENT_BIAS_CORRECTED_VERSION) or {}
|
||||
bucket = versions.get(DEB_BUCKET_CALIBRATED_VERSION) or {}
|
||||
guarded = versions.get(DEB_GUARDED_CALIBRATED_VERSION) or {}
|
||||
writer.writerow(
|
||||
{
|
||||
"city": row.get("city"),
|
||||
@@ -430,5 +668,23 @@ def write_backtest_report(
|
||||
f"{DEB_BUCKET_CALIBRATED_VERSION}_train_samples": bucket.get(
|
||||
"train_samples"
|
||||
),
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_prediction": guarded.get(
|
||||
"prediction"
|
||||
),
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_error": guarded.get(
|
||||
"error"
|
||||
),
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_bias_adjustment": guarded.get(
|
||||
"bias_adjustment"
|
||||
),
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_train_samples": guarded.get(
|
||||
"train_samples"
|
||||
),
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_selected_version": guarded.get(
|
||||
"selected_version"
|
||||
),
|
||||
f"{DEB_GUARDED_CALIBRATED_VERSION}_guard_reason": guarded.get(
|
||||
"guard_reason"
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
@@ -448,6 +448,7 @@ def analyze_weather_trend(
|
||||
deb_version = None
|
||||
deb_bias_adjustment = 0.0
|
||||
deb_bias_samples = 0
|
||||
deb_selected_version, deb_guard_reason = None, None
|
||||
deb_weights = ""
|
||||
deb_quality = {}
|
||||
if city_name and current_forecasts:
|
||||
@@ -460,6 +461,8 @@ def analyze_weather_trend(
|
||||
deb_prediction = deb_result.get("prediction")
|
||||
deb_raw_prediction = deb_result.get("raw_prediction")
|
||||
deb_version = deb_result.get("version")
|
||||
deb_selected_version = deb_result.get("selected_version")
|
||||
deb_guard_reason = deb_result.get("guard_reason")
|
||||
deb_bias_adjustment = deb_result.get("bias_adjustment") or 0.0
|
||||
deb_bias_samples = deb_result.get("bias_samples") or 0
|
||||
deb_weights = deb_result.get("weights_info") or ""
|
||||
@@ -998,6 +1001,8 @@ def analyze_weather_trend(
|
||||
"deb_raw_prediction": deb_raw_prediction,
|
||||
"deb_hourly_consensus": deb_hourly_consensus,
|
||||
"deb_version": deb_version,
|
||||
"deb_selected_version": deb_selected_version,
|
||||
"deb_guard_reason": deb_guard_reason,
|
||||
"deb_bias_adjustment": deb_bias_adjustment,
|
||||
"deb_bias_samples": deb_bias_samples,
|
||||
"deb_weights": deb_weights,
|
||||
|
||||
@@ -5,11 +5,13 @@ from pathlib import Path
|
||||
|
||||
from src.analysis.deb_evaluation import (
|
||||
DEB_BUCKET_CALIBRATED_VERSION,
|
||||
DEB_GUARDED_CALIBRATED_VERSION,
|
||||
DEB_RAW_VERSION,
|
||||
DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||
backtest_deb_versions,
|
||||
build_bucket_calibrated_corrector,
|
||||
build_recent_bias_corrector,
|
||||
choose_guarded_deb_correction,
|
||||
evaluate_prediction_records,
|
||||
write_backtest_report,
|
||||
)
|
||||
@@ -69,6 +71,58 @@ def test_bucket_calibrated_corrector_optimizes_settlement_bucket_hits():
|
||||
assert corrected["samples"] == 5
|
||||
|
||||
|
||||
def test_guarded_deb_correction_rejects_bucket_when_recent_holdout_gets_worse():
|
||||
history = [
|
||||
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-21", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-22", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-23", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-24", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-25", "deb_prediction": 20.49, "actual_high": 20.49},
|
||||
{"city": "ankara", "target_date": "2026-05-26", "deb_prediction": 20.49, "actual_high": 20.49},
|
||||
{"city": "ankara", "target_date": "2026-05-27", "deb_prediction": 20.49, "actual_high": 20.49},
|
||||
]
|
||||
|
||||
corrected = choose_guarded_deb_correction(
|
||||
history,
|
||||
"ankara",
|
||||
raw_prediction=20.49,
|
||||
min_samples=3,
|
||||
validation_samples=3,
|
||||
)
|
||||
|
||||
assert corrected["version"] == DEB_GUARDED_CALIBRATED_VERSION
|
||||
assert corrected["selected_version"] == DEB_RECENT_BIAS_CORRECTED_VERSION
|
||||
assert corrected["corrected_prediction"] == 20.5
|
||||
assert corrected["guard_reason"] == "bucket_rejected_holdout"
|
||||
|
||||
|
||||
def test_guarded_deb_correction_accepts_bucket_when_holdout_improves():
|
||||
history = [
|
||||
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-21", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-22", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-23", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-24", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-25", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-26", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
{"city": "ankara", "target_date": "2026-05-27", "deb_prediction": 20.49, "actual_high": 20.51},
|
||||
]
|
||||
|
||||
corrected = choose_guarded_deb_correction(
|
||||
history,
|
||||
"ankara",
|
||||
raw_prediction=20.49,
|
||||
min_samples=3,
|
||||
validation_samples=3,
|
||||
)
|
||||
|
||||
assert corrected["version"] == DEB_GUARDED_CALIBRATED_VERSION
|
||||
assert corrected["selected_version"] == DEB_BUCKET_CALIBRATED_VERSION
|
||||
assert corrected["corrected_prediction"] == 20.6
|
||||
assert corrected["guard_reason"] == "bucket_selected_holdout"
|
||||
|
||||
|
||||
def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions():
|
||||
history = [
|
||||
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.0, "actual_high": 22.0},
|
||||
@@ -83,6 +137,7 @@ def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions():
|
||||
assert report["versions"][DEB_RAW_VERSION]["samples"] == 2
|
||||
assert report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["samples"] == 2
|
||||
assert report["versions"][DEB_BUCKET_CALIBRATED_VERSION]["samples"] == 0
|
||||
assert report["versions"][DEB_GUARDED_CALIBRATED_VERSION]["samples"] == 2
|
||||
assert (
|
||||
report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["mae"]
|
||||
< report["versions"][DEB_RAW_VERSION]["mae"]
|
||||
|
||||
@@ -123,7 +123,9 @@ def test_calculate_deb_prediction_keeps_raw_and_adds_versioned_bias_correction(m
|
||||
|
||||
assert result["raw_prediction"] == 24.0
|
||||
assert result["prediction"] == 25.0
|
||||
assert result["version"] == "deb_v1_recent_bias_corrected"
|
||||
assert result["version"] == "deb_v3_guarded_calibrated"
|
||||
assert result["selected_version"] == "deb_v1_recent_bias_corrected"
|
||||
assert result["guard_reason"] == "bucket_unavailable"
|
||||
assert result["bias_adjustment"] == 1.0
|
||||
assert result["bias_samples"] == 3
|
||||
|
||||
@@ -169,7 +171,9 @@ def test_calculate_deb_prediction_prefers_bucket_calibration_when_enough_samples
|
||||
|
||||
assert result["raw_prediction"] == 25.4
|
||||
assert result["prediction"] == 26.0
|
||||
assert result["version"] == "deb_v2_bucket_calibrated"
|
||||
assert result["version"] == "deb_v3_guarded_calibrated"
|
||||
assert result["selected_version"] == "deb_v2_bucket_calibrated"
|
||||
assert result["guard_reason"] == "bucket_same_adjustment"
|
||||
assert result["bias_adjustment"] == 0.6
|
||||
assert result["bias_samples"] == 5
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ def test_training_accuracy_payload_includes_recent_deb_summary():
|
||||
assert payload["deb_summary"]["recent_14d"]["samples"] == 5
|
||||
assert "deb_v1_raw" in payload["deb_summary"]["versions"]
|
||||
assert "deb_v2_bucket_calibrated" in payload["deb_summary"]["versions"]
|
||||
assert "deb_v3_guarded_calibrated" in payload["deb_summary"]["versions"]
|
||||
|
||||
|
||||
def test_training_accuracy_payload_includes_city_deb_trust_strategy():
|
||||
|
||||
@@ -1196,6 +1196,7 @@ def _analyze(
|
||||
deb_val, deb_weights = None, ""
|
||||
deb_raw_val, deb_version = None, None
|
||||
deb_bias_adjustment, deb_bias_samples = 0.0, 0
|
||||
deb_selected_version, deb_guard_reason = None, None
|
||||
deb_intraday_adjustment = 0.0
|
||||
deb_hourly_consensus = None
|
||||
deb_quality = {}
|
||||
@@ -1205,6 +1206,8 @@ def _analyze(
|
||||
deb_val = deb_result.get("prediction")
|
||||
deb_raw_val = deb_result.get("raw_prediction")
|
||||
deb_version = deb_result.get("version")
|
||||
deb_selected_version = deb_result.get("selected_version")
|
||||
deb_guard_reason = deb_result.get("guard_reason")
|
||||
deb_bias_adjustment = deb_result.get("bias_adjustment") or 0.0
|
||||
deb_bias_samples = deb_result.get("bias_samples") or 0
|
||||
deb_weights = deb_result.get("weights_info") or ""
|
||||
@@ -1666,6 +1669,8 @@ def _analyze(
|
||||
d_val, d_winfo = deb_val, deb_weights
|
||||
d_raw_val = deb_raw_val
|
||||
d_version = deb_version
|
||||
d_selected_version = deb_selected_version
|
||||
d_guard_reason = deb_guard_reason
|
||||
d_bias_adjustment = deb_bias_adjustment
|
||||
d_bias_samples = deb_bias_samples
|
||||
d_quality = dict(deb_quality)
|
||||
@@ -1685,6 +1690,7 @@ def _analyze(
|
||||
|
||||
d_val, d_winfo = None, ""
|
||||
d_raw_val, d_version = None, None
|
||||
d_selected_version, d_guard_reason = None, None
|
||||
d_bias_adjustment, d_bias_samples = 0.0, 0
|
||||
d_quality = {}
|
||||
d_probs = []
|
||||
@@ -1697,6 +1703,8 @@ def _analyze(
|
||||
d_val = d_prediction
|
||||
d_raw_val = deb_result.get("raw_prediction")
|
||||
d_version = deb_result.get("version")
|
||||
d_selected_version = deb_result.get("selected_version")
|
||||
d_guard_reason = deb_result.get("guard_reason")
|
||||
d_bias_adjustment = deb_result.get("bias_adjustment") or 0.0
|
||||
d_bias_samples = deb_result.get("bias_samples") or 0
|
||||
d_winfo = deb_result.get("weights_info") or ""
|
||||
@@ -1731,6 +1739,8 @@ def _analyze(
|
||||
"prediction": d_val,
|
||||
"raw_prediction": d_raw_val,
|
||||
"version": d_version,
|
||||
"selected_version": d_selected_version,
|
||||
"guard_reason": d_guard_reason,
|
||||
"weights_info": d_winfo,
|
||||
"bias_adjustment": d_bias_adjustment,
|
||||
"bias_samples": d_bias_samples,
|
||||
@@ -2223,6 +2233,7 @@ def _analyze_summary(city: str, force_refresh: bool = False) -> Dict[str, Any]:
|
||||
deb_version = None
|
||||
deb_bias_adjustment = 0.0
|
||||
deb_bias_samples = 0
|
||||
deb_selected_version, deb_guard_reason = None, None
|
||||
deb_quality = {}
|
||||
if current_forecasts:
|
||||
deb_result = calculate_deb_prediction(city, current_forecasts)
|
||||
@@ -2230,6 +2241,8 @@ def _analyze_summary(city: str, force_refresh: bool = False) -> Dict[str, Any]:
|
||||
deb_val = deb_result.get("prediction")
|
||||
deb_raw_val = deb_result.get("raw_prediction")
|
||||
deb_version = deb_result.get("version")
|
||||
deb_selected_version = deb_result.get("selected_version")
|
||||
deb_guard_reason = deb_result.get("guard_reason")
|
||||
deb_bias_adjustment = deb_result.get("bias_adjustment") or 0.0
|
||||
deb_bias_samples = deb_result.get("bias_samples") or 0
|
||||
deb_quality = {
|
||||
@@ -2314,6 +2327,8 @@ def _analyze_summary(city: str, force_refresh: bool = False) -> Dict[str, Any]:
|
||||
"prediction": _sf(deb_val),
|
||||
"raw_prediction": _sf(deb_raw_val),
|
||||
"version": deb_version,
|
||||
"selected_version": deb_selected_version,
|
||||
"guard_reason": deb_guard_reason,
|
||||
"bias_adjustment": deb_bias_adjustment,
|
||||
"bias_samples": deb_bias_samples,
|
||||
**deb_quality,
|
||||
|
||||
Reference in New Issue
Block a user