diff --git a/frontend/components/dashboard/scan-terminal/TrainingDashboard.tsx b/frontend/components/dashboard/scan-terminal/TrainingDashboard.tsx index 9582d17e..65ac737c 100644 --- a/frontend/components/dashboard/scan-terminal/TrainingDashboard.tsx +++ b/frontend/components/dashboard/scan-terminal/TrainingDashboard.tsx @@ -19,6 +19,43 @@ type MetricPayload = { brier_score?: number; }; +type DebWindowSummary = { + start_date?: string | null; + end_date?: string | null; + samples?: number; + hits?: number; + hit_rate?: number | null; + mae?: number | null; + bias?: number | null; + city_count?: number; +}; + +type DebHistoricalSummary = { + city_count?: number; + avg_hit_rate?: number | null; + weighted_hit_rate?: number | null; + avg_mae?: number | null; + avg_days_per_city?: number; + sample_days?: number; + hits?: number; +}; + +type DebVersionSummary = { + version?: string; + samples?: number; + mae?: number | null; + rmse?: number | null; + bias?: number | null; + bucket_hit_rate?: number | null; +}; + +type DebSummaryPayload = { + historical?: DebHistoricalSummary; + recent_7d?: DebWindowSummary; + recent_14d?: DebWindowSummary; + versions?: Record; +}; + type TrainingCity = { city_id: string; name: string; @@ -26,6 +63,11 @@ type TrainingCity = { mu?: MetricPayload; }; +type TrainingAccuracyPayload = { + accuracy: TrainingCity[]; + deb_summary?: DebSummaryPayload; +}; + const STAT_CARD_CLASSES: Record = { blue: "bg-blue-50 border-blue-200", emerald: "bg-emerald-50 border-emerald-200", @@ -48,44 +90,48 @@ function barColor(hr: number) { const TRAINING_CACHE_KEY = "polyweather_training_accuracy_v1"; const TRAINING_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours -function readTrainingCache(): TrainingCity[] | null { +function readTrainingCache(): TrainingAccuracyPayload | null { try { const raw = localStorage.getItem(TRAINING_CACHE_KEY); if (!raw) return null; const cached = JSON.parse(raw); - if (cached.ts && Date.now() - cached.ts < TRAINING_CACHE_TTL_MS && Array.isArray(cached.data)) { - return cached.data; + if (cached.ts && Date.now() - cached.ts < TRAINING_CACHE_TTL_MS) { + if (Array.isArray(cached.data)) return { accuracy: cached.data }; + if (cached.data && Array.isArray(cached.data.accuracy)) return cached.data; } } catch { /* ignore */ } return null; } -function writeTrainingCache(data: TrainingCity[]) { +function writeTrainingCache(data: TrainingAccuracyPayload) { try { localStorage.setItem(TRAINING_CACHE_KEY, JSON.stringify({ ts: Date.now(), data })); } catch { /* ignore */ } } export function TrainingDashboard({ isEn }: { isEn: boolean }) { - const [data, setData] = useState(() => readTrainingCache()); + const [payload, setPayload] = useState(() => readTrainingCache()); useEffect(() => { let cancelled = false; fetch("/api/ops/training/accuracy", { cache: "no-store", headers: { Accept: "application/json" } }) .then(async (res) => { if (!res.ok) return null; - return res.json() as Promise<{ accuracy: TrainingCity[] }>; + return res.json() as Promise; }) - .then((payload) => { - if (cancelled || !payload?.accuracy) return; - const filtered = payload.accuracy.filter((c) => (c.deb || c.mu) && ((c.deb?.total_days ?? 0) + (c.mu?.total_days ?? 0)) >= 5); - setData(filtered); - writeTrainingCache(filtered); + .then((nextPayload) => { + if (cancelled || !nextPayload?.accuracy) return; + const filtered = nextPayload.accuracy.filter((c) => (c.deb || c.mu) && ((c.deb?.total_days ?? 0) + (c.mu?.total_days ?? 0)) >= 5); + const next = { ...nextPayload, accuracy: filtered }; + setPayload(next); + writeTrainingCache(next); }) .catch(() => {}); return () => { cancelled = true; }; }, []); + const data = payload?.accuracy ?? null; + const debSummary = payload?.deb_summary; const debSorted = useMemo(() => (data || []).filter((c) => c.deb).sort((a, b) => (b.deb?.hit_rate ?? 0) - (a.deb?.hit_rate ?? 0)), [data]); const muSorted = useMemo(() => (data || []).filter((c) => c.mu).sort((a, b) => (b.mu?.hit_rate ?? 0) - (a.mu?.hit_rate ?? 0)), [data]); @@ -94,8 +140,15 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) { const avgHit = debSorted.reduce((s, c) => s + (c.deb?.hit_rate ?? 0), 0) / debSorted.length; const avgMae = debSorted.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debSorted.length; const avgDays = Math.round(debSorted.reduce((s, c) => s + (c.deb?.total_days ?? 0), 0) / Math.max(debSorted.length, 1)); - return { avgHit, avgMae, avgDays, cities: debSorted.length }; - }, [debSorted]); + return { + avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit, + avgMae: debSummary?.historical?.avg_mae ?? avgMae, + avgDays: debSummary?.historical?.avg_days_per_city ?? avgDays, + cities: debSummary?.historical?.city_count ?? debSorted.length, + sampleDays: debSummary?.historical?.sample_days, + weightedHit: debSummary?.historical?.weighted_hit_rate, + }; + }, [debSorted, debSummary]); const muStats = useMemo(() => { if (!muSorted.length) return null; @@ -122,6 +175,17 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) { () => [...muSorted].sort((a, b) => (a.mu?.brier_score ?? 99) - (b.mu?.brier_score ?? 99)).slice(0, 18).map((c) => ({ name: c.name, value: Number((c.mu?.brier_score ?? 0).toFixed(3)) })), [muSorted], ); + const debVersionRows = useMemo(() => { + const versions = debSummary?.versions || {}; + return [ + { key: "deb_v1_raw", label: isEn ? "Raw DEB" : "原始 DEB" }, + { key: "deb_v1_recent_bias_corrected", label: isEn ? "Mean Bias" : "均值偏差" }, + { key: "deb_v2_bucket_calibrated", label: isEn ? "Bucket v2" : "桶校准 v2" }, + ].map(({ key, label }) => ({ key, label, value: versions[key] })).filter((row) => row.value); + }, [debSummary?.versions, isEn]); + + const formatPct = (value: number | null | undefined) => value == null ? "--" : `${value.toFixed(1)}%`; + const formatMaybeDeg = (value: number | null | undefined, digits = 1) => value == null ? "--" : `${value.toFixed(digits)}°`; return (
@@ -143,12 +207,14 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) { {isEn ? "DEB Temperature Forecast" : "DEB 气温预报"} -
+
{[ { icon: Hash, label: isEn ? "Cities" : "城市数", value: debStats.cities, tone: "blue" }, - { icon: Target, label: isEn ? "Avg Hit" : "平均命中", value: `${debStats.avgHit.toFixed(1)}%`, tone: "emerald" }, + { icon: Target, label: isEn ? "Historical Avg" : "历史平均", value: `${debStats.avgHit.toFixed(1)}%`, tone: "emerald" }, + { icon: TrendingUp, label: isEn ? "Recent 7d" : "近7天", value: formatPct(debSummary?.recent_7d?.hit_rate), tone: "emerald" }, + { icon: TrendingUp, label: isEn ? "Recent 14d" : "近14天", value: formatPct(debSummary?.recent_14d?.hit_rate), tone: "purple" }, { icon: Thermometer, label: isEn ? "Avg Error" : "平均误差", value: `${debStats.avgMae.toFixed(1)}°`, tone: "amber" }, - { icon: TrendingUp, label: isEn ? "Avg Days/City" : "每城平均天数", value: debStats.avgDays.toLocaleString(), tone: "purple" }, + { icon: Hash, label: isEn ? "Samples" : "样本天数", value: (debStats.sampleDays ?? debStats.avgDays).toLocaleString(), tone: "blue" }, ].map(({ icon: Icon, label, value, tone }) => (
@@ -159,6 +225,25 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
))}
+ {debVersionRows.length ? ( +
+ {debVersionRows.map((row) => { + const bucketRate = row.value?.bucket_hit_rate == null ? null : row.value.bucket_hit_rate * 100; + return ( +
+
+ {row.label} + {formatPct(bucketRate)} +
+
+ {isEn ? "MAE" : "误差"} {formatMaybeDeg(row.value?.mae, 2)} + {isEn ? "Samples" : "样本"} {row.value?.samples ?? 0} +
+
+ ); + })} +
+ ) : null}
diff --git a/frontend/components/ops/training/TrainingPageClient.tsx b/frontend/components/ops/training/TrainingPageClient.tsx index fc3957b4..df761cfb 100644 --- a/frontend/components/ops/training/TrainingPageClient.tsx +++ b/frontend/components/ops/training/TrainingPageClient.tsx @@ -65,20 +65,44 @@ interface CityAccuracy { } | null; } +interface DebSummary { + historical?: { + avg_hit_rate?: number | null; + weighted_hit_rate?: number | null; + avg_mae?: number | null; + sample_days?: number; + city_count?: number; + }; + recent_7d?: { + hit_rate?: number | null; + mae?: number | null; + samples?: number; + hits?: number; + }; + recent_14d?: { + hit_rate?: number | null; + mae?: number | null; + samples?: number; + hits?: number; + }; +} + export function TrainingPageClient() { const [loading, setLoading] = useState(true); const [status, setStatus] = useState(null); const [accuracy, setAccuracy] = useState(null); + const [debSummary, setDebSummary] = useState(null); const load = async () => { setLoading(true); try { const [s, accData] = await Promise.all([ opsApi.systemStatus() as Promise, - opsApi.trainingAccuracy().catch(() => ({ accuracy: [] as CityAccuracy[] })), + opsApi.trainingAccuracy().catch(() => ({ accuracy: [] as CityAccuracy[], deb_summary: null })), ]); setStatus(s); setAccuracy((accData as { accuracy: CityAccuracy[] }).accuracy ?? []); + setDebSummary((accData as { deb_summary?: DebSummary | null }).deb_summary ?? null); } catch { /* */ } setLoading(false); }; @@ -92,8 +116,15 @@ export function TrainingPageClient() { const avgMae = debCities.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debCities.length; const best = debCities.reduce((a, b) => ((a.deb?.hit_rate ?? 0) > (b.deb?.hit_rate ?? 0) ? a : b)); const worst = debCities.reduce((a, b) => ((a.deb?.mae ?? 0) > (b.deb?.mae ?? 0) ? a : b)); - return { avgHit, avgMae, best, worst }; - }, [accuracy]); + return { + avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit, + avgMae: debSummary?.historical?.avg_mae ?? avgMae, + recent7Hit: debSummary?.recent_7d?.hit_rate, + recent14Hit: debSummary?.recent_14d?.hit_rate, + best, + worst, + }; + }, [accuracy, debSummary]); const debChartData = useMemo(() => { if (!accuracy?.length) return []; @@ -172,10 +203,18 @@ export function TrainingPageClient() { {/* Accuracy KPI row */} {kpis ? ( -
+
+ + ; + deb_summary?: { + historical?: { + city_count?: number; + avg_hit_rate?: number | null; + weighted_hit_rate?: number | null; + avg_mae?: number | null; + avg_days_per_city?: number; + sample_days?: number; + hits?: number; + }; + recent_7d?: { + start_date?: string | null; + end_date?: string | null; + samples?: number; + hits?: number; + hit_rate?: number | null; + mae?: number | null; + bias?: number | null; + city_count?: number; + }; + recent_14d?: { + start_date?: string | null; + end_date?: string | null; + samples?: number; + hits?: number; + hit_rate?: number | null; + mae?: number | null; + bias?: number | null; + city_count?: number; + }; + versions?: Record; + }; }>("/api/ops/training/accuracy"); }, telegramAudit() { diff --git a/src/analysis/deb_algorithm.py b/src/analysis/deb_algorithm.py index 64672aaa..76c0f342 100644 --- a/src/analysis/deb_algorithm.py +++ b/src/analysis/deb_algorithm.py @@ -1267,8 +1267,10 @@ def calculate_deb_prediction( adjustment when enough settled samples exist. """ from src.analysis.deb_evaluation import ( + DEB_BUCKET_CALIBRATED_VERSION, DEB_RAW_VERSION, DEB_RECENT_BIAS_CORRECTED_VERSION, + build_bucket_calibrated_corrector, build_recent_bias_corrector, flatten_daily_records, ) @@ -1297,6 +1299,14 @@ def calculate_deb_prediction( lookback_days=bias_lookback_days, min_samples=bias_min_samples, ).apply(city_name, raw_prediction) + bucket_corrected = build_bucket_calibrated_corrector( + history_rows, + lookback_days=bias_lookback_days, + min_samples=max(5, int(bias_min_samples or 0)), + ).apply(city_name, raw_prediction) + if int(bucket_corrected.get("samples") or 0) > 0: + corrected = bucket_corrected + bias_adjustment = float(corrected.get("bias_adjustment") or 0.0) bias_samples = int(corrected.get("samples") or 0) if bias_samples <= 0: @@ -1311,14 +1321,19 @@ def calculate_deb_prediction( next_weights_info = weights_info if abs(bias_adjustment) >= 0.05: + correction_label = ( + "bucket_calibration" + if corrected.get("version") == DEB_BUCKET_CALIBRATED_VERSION + else "recent_bias" + ) next_weights_info = ( f"{weights_info or 'DEB'} | " - f"recent_bias({bias_adjustment:+.1f},n={bias_samples})" + f"{correction_label}({bias_adjustment:+.1f},n={bias_samples})" ) return { "prediction": corrected["corrected_prediction"], "raw_prediction": corrected["raw_prediction"], - "version": DEB_RECENT_BIAS_CORRECTED_VERSION, + "version": corrected.get("version") or DEB_RECENT_BIAS_CORRECTED_VERSION, "weights_info": next_weights_info, "bias_adjustment": bias_adjustment, "bias_samples": bias_samples, diff --git a/src/analysis/deb_evaluation.py b/src/analysis/deb_evaluation.py index d4c8e441..62d885b6 100644 --- a/src/analysis/deb_evaluation.py +++ b/src/analysis/deb_evaluation.py @@ -12,6 +12,7 @@ from src.analysis.settlement_rounding import apply_city_settlement DEB_RAW_VERSION = "deb_v1_raw" DEB_RECENT_BIAS_CORRECTED_VERSION = "deb_v1_recent_bias_corrected" +DEB_BUCKET_CALIBRATED_VERSION = "deb_v2_bucket_calibrated" DEB_BACKTEST_SCHEMA_VERSION = "deb_backtest_report.v1" @@ -108,8 +109,14 @@ class BiasCorrectionResult: class RecentBiasCorrector: - def __init__(self, bias_by_city: dict[str, tuple[float, int]]) -> None: + def __init__( + self, + bias_by_city: dict[str, tuple[float, int]], + *, + version: str = DEB_RECENT_BIAS_CORRECTED_VERSION, + ) -> None: self._bias_by_city = bias_by_city + self._version = version def apply(self, city: str, raw_prediction: float) -> dict[str, Any]: city_key = str(city or "").strip().lower() @@ -117,7 +124,7 @@ class RecentBiasCorrector: bias, samples = self._bias_by_city.get(city_key, (0.0, 0)) adjustment = round(bias, 1) return BiasCorrectionResult( - version=DEB_RECENT_BIAS_CORRECTED_VERSION, + version=self._version, raw_prediction=round(raw, 1), corrected_prediction=round(raw + adjustment, 1), bias_adjustment=adjustment, @@ -159,6 +166,71 @@ def build_recent_bias_corrector( return RecentBiasCorrector(bias_by_city) +def build_bucket_calibrated_corrector( + history: Iterable[dict[str, Any]], + *, + lookback_days: int = 30, + min_samples: int = 5, + max_adjustment: float = 3.0, + step: float = 0.1, +) -> RecentBiasCorrector: + by_city: dict[str, list[dict[str, Any]]] = {} + for record in history: + row = _normalise_record(record) + if row is None: + continue + by_city.setdefault(row["city"], []).append(row) + + adjustment_by_city: dict[str, tuple[float, int]] = {} + safe_step = max(abs(float(step or 0.1)), 0.1) + max_abs = abs(float(max_adjustment or 0.0)) + candidate_count = int(round((max_abs * 2) / safe_step)) + 1 + candidates = [ + round(-max_abs + idx * safe_step, 1) + for idx in range(max(candidate_count, 1)) + ] + + for city, rows in by_city.items(): + rows.sort(key=lambda row: row["target_date"], reverse=True) + recent = rows[: max(int(lookback_days or 0), 1)] + if len(recent) < min_samples: + continue + + best = None + for adjustment in candidates: + hits = 0 + total = 0 + abs_errors: list[float] = [] + for row in recent: + prediction = row["prediction"] + adjustment + actual = row["actual"] + try: + pred_bucket = apply_city_settlement(city, prediction) + actual_bucket = apply_city_settlement(city, actual) + except Exception: + continue + if pred_bucket is None or actual_bucket is None: + continue + total += 1 + if pred_bucket == actual_bucket: + hits += 1 + abs_errors.append(abs(prediction - actual)) + if not total: + continue + mae = statistics.mean(abs_errors) if abs_errors else float("inf") + score = (hits, -mae, -abs(adjustment), adjustment) + if best is None or score > best: + best = score + + if best is not None: + adjustment_by_city[city] = (best[3], len(recent)) + + return RecentBiasCorrector( + adjustment_by_city, + version=DEB_BUCKET_CALIBRATED_VERSION, + ) + + def backtest_deb_versions( history: Iterable[dict[str, Any]], *, @@ -171,6 +243,7 @@ def backtest_deb_versions( report_rows: list[dict[str, Any]] = [] raw_eval_rows: list[dict[str, Any]] = [] corrected_eval_rows: list[dict[str, Any]] = [] + bucket_eval_rows: list[dict[str, Any]] = [] by_city: dict[str, list[dict[str, Any]]] = {} for row in rows: @@ -181,9 +254,15 @@ def backtest_deb_versions( lookback_days=train_lookback_days, min_samples=min_train_samples, ) + bucket_corrector = build_bucket_calibrated_corrector( + previous, + lookback_days=train_lookback_days, + ) corrected = corrector.apply(row["city"], row["prediction"]) + bucket_corrected = bucket_corrector.apply(row["city"], row["prediction"]) raw_prediction = round(row["prediction"], 1) corrected_prediction = corrected["corrected_prediction"] + bucket_prediction = bucket_corrected["corrected_prediction"] raw_eval_rows.append( { @@ -201,6 +280,15 @@ def backtest_deb_versions( "actual": row["actual"], } ) + if int(bucket_corrected.get("samples") or 0) > 0: + bucket_eval_rows.append( + { + "city": row["city"], + "target_date": row["target_date"], + "prediction": bucket_prediction, + "actual": row["actual"], + } + ) report_rows.append( { "city": row["city"], @@ -217,6 +305,12 @@ def backtest_deb_versions( "bias_adjustment": corrected["bias_adjustment"], "train_samples": corrected["samples"], }, + DEB_BUCKET_CALIBRATED_VERSION: { + "prediction": bucket_prediction, + "error": round(bucket_prediction - row["actual"], 3), + "bias_adjustment": bucket_corrected["bias_adjustment"], + "train_samples": bucket_corrected["samples"], + }, }, } ) @@ -233,6 +327,10 @@ def backtest_deb_versions( corrected_eval_rows, version=DEB_RECENT_BIAS_CORRECTED_VERSION, ), + DEB_BUCKET_CALIBRATED_VERSION: evaluate_prediction_records( + bucket_eval_rows, + version=DEB_BUCKET_CALIBRATED_VERSION, + ), }, "rows": report_rows, } @@ -289,6 +387,10 @@ def write_backtest_report( f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_error", f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_bias_adjustment", f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_train_samples", + f"{DEB_BUCKET_CALIBRATED_VERSION}_prediction", + f"{DEB_BUCKET_CALIBRATED_VERSION}_error", + f"{DEB_BUCKET_CALIBRATED_VERSION}_bias_adjustment", + f"{DEB_BUCKET_CALIBRATED_VERSION}_train_samples", ], ) writer.writeheader() @@ -296,6 +398,7 @@ def write_backtest_report( versions = row.get("versions") or {} raw = versions.get(DEB_RAW_VERSION) or {} corrected = versions.get(DEB_RECENT_BIAS_CORRECTED_VERSION) or {} + bucket = versions.get(DEB_BUCKET_CALIBRATED_VERSION) or {} writer.writerow( { "city": row.get("city"), @@ -315,5 +418,17 @@ def write_backtest_report( f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_train_samples": corrected.get( "train_samples" ), + f"{DEB_BUCKET_CALIBRATED_VERSION}_prediction": bucket.get( + "prediction" + ), + f"{DEB_BUCKET_CALIBRATED_VERSION}_error": bucket.get( + "error" + ), + f"{DEB_BUCKET_CALIBRATED_VERSION}_bias_adjustment": bucket.get( + "bias_adjustment" + ), + f"{DEB_BUCKET_CALIBRATED_VERSION}_train_samples": bucket.get( + "train_samples" + ), } ) diff --git a/tests/test_deb_evaluation_upgrade.py b/tests/test_deb_evaluation_upgrade.py index 77a99090..cde8c0e8 100644 --- a/tests/test_deb_evaluation_upgrade.py +++ b/tests/test_deb_evaluation_upgrade.py @@ -4,9 +4,11 @@ import sys from pathlib import Path from src.analysis.deb_evaluation import ( + DEB_BUCKET_CALIBRATED_VERSION, DEB_RAW_VERSION, DEB_RECENT_BIAS_CORRECTED_VERSION, backtest_deb_versions, + build_bucket_calibrated_corrector, build_recent_bias_corrector, evaluate_prediction_records, write_backtest_report, @@ -48,6 +50,25 @@ def test_recent_bias_corrector_uses_signed_error_without_rewriting_raw_deb(): assert corrected["samples"] == 3 +def test_bucket_calibrated_corrector_optimizes_settlement_bucket_hits(): + history = [ + {"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.4, "actual_high": 21.0}, + {"city": "ankara", "target_date": "2026-05-21", "deb_prediction": 21.4, "actual_high": 22.0}, + {"city": "ankara", "target_date": "2026-05-22", "deb_prediction": 22.4, "actual_high": 23.0}, + {"city": "ankara", "target_date": "2026-05-23", "deb_prediction": 23.4, "actual_high": 24.0}, + {"city": "ankara", "target_date": "2026-05-24", "deb_prediction": 24.4, "actual_high": 25.0}, + ] + + corrector = build_bucket_calibrated_corrector(history, lookback_days=30, min_samples=5) + corrected = corrector.apply("ankara", raw_prediction=25.4) + + assert corrected["version"] == DEB_BUCKET_CALIBRATED_VERSION + assert corrected["raw_prediction"] == 25.4 + assert corrected["corrected_prediction"] == 26.0 + assert corrected["bias_adjustment"] == 0.6 + assert corrected["samples"] == 5 + + def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions(): history = [ {"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.0, "actual_high": 22.0}, @@ -61,6 +82,7 @@ def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions(): assert report["schema_version"] == "deb_backtest_report.v1" assert report["versions"][DEB_RAW_VERSION]["samples"] == 2 assert report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["samples"] == 2 + assert report["versions"][DEB_BUCKET_CALIBRATED_VERSION]["samples"] == 0 assert ( report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["mae"] < report["versions"][DEB_RAW_VERSION]["mae"] diff --git a/tests/test_deb_model_family.py b/tests/test_deb_model_family.py index b372678b..79120961 100644 --- a/tests/test_deb_model_family.py +++ b/tests/test_deb_model_family.py @@ -128,6 +128,52 @@ def test_calculate_deb_prediction_keeps_raw_and_adds_versioned_bias_correction(m assert result["bias_samples"] == 3 +def test_calculate_deb_prediction_prefers_bucket_calibration_when_enough_samples(monkeypatch): + monkeypatch.setattr( + "src.analysis.deb_algorithm.load_history", + lambda _: { + "ankara": { + "2026-04-11": { + "actual_high": 21.0, + "deb_prediction": 20.4, + "forecasts": {"ECMWF": 20.4, "GFS": 20.4}, + }, + "2026-04-12": { + "actual_high": 22.0, + "deb_prediction": 21.4, + "forecasts": {"ECMWF": 21.4, "GFS": 21.4}, + }, + "2026-04-13": { + "actual_high": 23.0, + "deb_prediction": 22.4, + "forecasts": {"ECMWF": 22.4, "GFS": 22.4}, + }, + "2026-04-14": { + "actual_high": 24.0, + "deb_prediction": 23.4, + "forecasts": {"ECMWF": 23.4, "GFS": 23.4}, + }, + "2026-04-15": { + "actual_high": 25.0, + "deb_prediction": 24.4, + "forecasts": {"ECMWF": 24.4, "GFS": 24.4}, + }, + } + }, + ) + + result = calculate_deb_prediction( + "ankara", + {"ECMWF": 25.4, "GFS": 25.4}, + ) + + assert result["raw_prediction"] == 25.4 + assert result["prediction"] == 26.0 + assert result["version"] == "deb_v2_bucket_calibrated" + assert result["bias_adjustment"] == 0.6 + assert result["bias_samples"] == 5 + + def test_compute_hourly_model_errors_basic(): from src.analysis.deb_algorithm import compute_hourly_model_errors diff --git a/tests/test_ops_training_accuracy.py b/tests/test_ops_training_accuracy.py new file mode 100644 index 00000000..3dfa2426 --- /dev/null +++ b/tests/test_ops_training_accuracy.py @@ -0,0 +1,38 @@ +from web.services.ops_api import _build_training_accuracy_payload + + +def test_training_accuracy_payload_includes_recent_deb_summary(): + history = { + "alpha": { + "2026-05-25": {"actual_high": 20.0, "deb_prediction": 20.0, "mu": 20.0}, + "2026-06-01": {"actual_high": 21.0, "deb_prediction": 20.4, "mu": 21.0}, + "2026-06-02": {"actual_high": 22.0, "deb_prediction": 21.6, "mu": 22.0}, + }, + "beta": { + "2026-06-03": {"actual_high": 30.0, "deb_prediction": 30.2, "mu": 30.0}, + "2026-06-04": {"actual_high": 31.0, "deb_prediction": 29.2, "mu": 31.0}, + }, + } + registry = { + "alpha": {"name": "Alpha"}, + "beta": {"name": "Beta"}, + "gamma": {"name": "Gamma"}, + } + + payload = _build_training_accuracy_payload( + history, + registry, + today_str="2026-06-07", + ) + + assert [row["city_id"] for row in payload["accuracy"]] == ["alpha", "beta"] + assert payload["deb_summary"]["historical"]["city_count"] == 2 + assert payload["deb_summary"]["historical"]["sample_days"] == 5 + assert payload["deb_summary"]["recent_7d"]["start_date"] == "2026-05-31" + assert payload["deb_summary"]["recent_7d"]["end_date"] == "2026-06-06" + assert payload["deb_summary"]["recent_7d"]["samples"] == 4 + assert payload["deb_summary"]["recent_7d"]["hits"] == 2 + assert payload["deb_summary"]["recent_7d"]["hit_rate"] == 50.0 + assert payload["deb_summary"]["recent_14d"]["samples"] == 5 + assert "deb_v1_raw" in payload["deb_summary"]["versions"] + assert "deb_v2_bucket_calibrated" in payload["deb_summary"]["versions"] diff --git a/web/services/ops_api.py b/web/services/ops_api.py index 16fc6979..3d4e42f5 100644 --- a/web/services/ops_api.py +++ b/web/services/ops_api.py @@ -19,6 +19,19 @@ from web.core import GrantPointsRequest import web.routes as legacy_routes +def _sf(value: Any) -> Optional[float]: + try: + if value is None or value == "": + return None + return float(value) + except (TypeError, ValueError): + return None + + +def _round_metric(value: Optional[float], digits: int = 1) -> Optional[float]: + return None if value is None else round(float(value), digits) + + def _require_ops(request: Request) -> Dict[str, Any] | None: # Ops admins are authenticated via Supabase identity + email whitelist. # They do NOT need an active Pro subscription to manage the system. @@ -2212,52 +2225,271 @@ def get_ops_health_check(request: Request) -> dict[str, Any]: } -def get_ops_training_accuracy(request: Request) -> Dict[str, Any]: - from src.analysis.deb_algorithm import get_deb_accuracy, get_mu_accuracy - from src.data_collection.city_registry import CITY_REGISTRY +def _evaluate_deb_records( + records: List[Dict[str, Any]], + *, + start_date: Optional[str] = None, + end_date: Optional[str] = None, +) -> Dict[str, Any]: + from src.analysis.settlement_rounding import apply_city_settlement - accuracy_data = [] - for city_id, info in CITY_REGISTRY.items(): - name = info.get("name") or city_id + hits = 0 + total = 0 + errors: List[float] = [] + signed_errors: List[float] = [] + cities = set() + dates = set() + for row in records: + target_date = str(row.get("target_date") or "").strip() + if start_date and target_date < start_date: + continue + if end_date and target_date > end_date: + continue + city = str(row.get("city") or "").strip().lower() + prediction = _sf(row.get("deb_prediction")) + actual = _sf(row.get("actual_high")) + if not city or not target_date or prediction is None or actual is None: + continue + try: + pred_bucket = apply_city_settlement(city, prediction) + actual_bucket = apply_city_settlement(city, actual) + except Exception: + continue + if pred_bucket is None or actual_bucket is None: + continue + total += 1 + if pred_bucket == actual_bucket: + hits += 1 + errors.append(abs(prediction - actual)) + signed_errors.append(prediction - actual) + cities.add(city) + dates.add(target_date) - # Calculate DEB accuracy - deb_acc = get_deb_accuracy(city_id) - deb_payload = None - if deb_acc: - deb_payload = { - "hit_rate": deb_acc[0], - "mae": deb_acc[1], - "total_days": deb_acc[2], - "details_str": deb_acc[3], + return { + "start_date": start_date, + "end_date": end_date, + "samples": total, + "hits": hits, + "hit_rate": _round_metric((hits / total * 100) if total else None, 1), + "mae": _round_metric((sum(errors) / len(errors)) if errors else None, 2), + "bias": _round_metric((sum(signed_errors) / len(signed_errors)) if signed_errors else None, 2), + "city_count": len(cities), + "date_count": len(dates), + } + + +def _build_city_deb_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]: + rows = [] + for target_date, record in sorted((city_rows or {}).items()): + if target_date >= today_str or not isinstance(record, dict): + continue + rows.append( + { + "city": city_id, + "target_date": target_date, + "actual_high": record.get("actual_high"), + "deb_prediction": record.get("deb_prediction"), } + ) + metrics = _evaluate_deb_records(rows) + if not metrics["samples"]: + return None + total = int(metrics["samples"]) + hits = int(metrics["hits"]) + mae = float(metrics["mae"] or 0.0) + hit_rate = float(metrics["hit_rate"] or 0.0) + return { + "hit_rate": hit_rate, + "mae": mae, + "total_days": total, + "hits": hits, + "details_str": f"过去{total}天 WU命中 {hits}/{total} ({hit_rate:.0f}%) | MAE: {mae:.1f}°", + } - # Calculate Mu accuracy - mu_acc = get_mu_accuracy(city_id) - mu_payload = None - if mu_acc: - mu_payload = { - "mae": mu_acc[0], - "hit_rate": mu_acc[1], - "brier_score": mu_acc[2], - "total_days": mu_acc[3], - "details_str": mu_acc[4], - } +def _build_city_mu_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]: + from src.analysis.settlement_rounding import apply_city_settlement + + errors: List[float] = [] + hits = 0 + total = 0 + brier_scores: List[float] = [] + for target_date, record in sorted((city_rows or {}).items()): + if target_date >= today_str or not isinstance(record, dict): + continue + actual = _sf(record.get("actual_high")) + mu_value = _sf(record.get("mu")) + if actual is None or mu_value is None: + continue + total += 1 + errors.append(abs(mu_value - actual)) + if apply_city_settlement(city_id, mu_value) == apply_city_settlement(city_id, actual): + hits += 1 + prob_snapshot = record.get("prob_snapshot") or [] + if isinstance(prob_snapshot, list): + actual_bucket = apply_city_settlement(city_id, actual) + score = 0.0 + used = False + for entry in prob_snapshot: + if not isinstance(entry, dict): + continue + predicted_p = _sf(entry.get("p")) or 0.0 + outcome = 1.0 if entry.get("v") == actual_bucket else 0.0 + score += (predicted_p - outcome) ** 2 + used = True + if used: + brier_scores.append(score) + + if not total: + return None + mae = sum(errors) / len(errors) + hit_rate = hits / total * 100 + brier = (sum(brier_scores) / len(brier_scores)) if brier_scores else None + details_parts = [ + f"μ准确率: 过去{total}天", + f"WU命中 {hits}/{total} ({hit_rate:.0f}%)", + f"MAE: {mae:.1f}°", + ] + if brier is not None: + details_parts.append(f"Brier: {brier:.3f}") + return { + "mae": mae, + "hit_rate": hit_rate, + "brier_score": brier, + "total_days": total, + "hits": hits, + "details_str": " | ".join(details_parts), + } + + +def _build_deb_historical_summary(accuracy_data: List[Dict[str, Any]]) -> Dict[str, Any]: + deb_rows = [row for row in accuracy_data if row.get("deb")] + if not deb_rows: + return { + "city_count": 0, + "avg_hit_rate": None, + "weighted_hit_rate": None, + "avg_mae": None, + "avg_days_per_city": 0, + "sample_days": 0, + "hits": 0, + } + sample_days = sum(int(row["deb"].get("total_days") or 0) for row in deb_rows) + hits = sum(int(row["deb"].get("hits") or 0) for row in deb_rows) + return { + "city_count": len(deb_rows), + "avg_hit_rate": _round_metric( + sum(float(row["deb"].get("hit_rate") or 0.0) for row in deb_rows) / len(deb_rows), + 1, + ), + "weighted_hit_rate": _round_metric((hits / sample_days * 100) if sample_days else None, 1), + "avg_mae": _round_metric( + sum(float(row["deb"].get("mae") or 0.0) for row in deb_rows) / len(deb_rows), + 2, + ), + "avg_days_per_city": round(sample_days / len(deb_rows)) if deb_rows else 0, + "sample_days": sample_days, + "hits": hits, + } + + +def _flatten_training_history(history: Dict[str, Dict[str, Dict[str, Any]]], today_str: str) -> List[Dict[str, Any]]: + rows: List[Dict[str, Any]] = [] + for city, city_rows in (history or {}).items(): + if not isinstance(city_rows, dict): + continue + for target_date, record in city_rows.items(): + if not isinstance(record, dict): + continue + target_text = str(target_date or "").strip() + if not target_text or target_text >= today_str: + continue + rows.append( + { + "city": str(city or "").strip().lower(), + "target_date": target_text, + "actual_high": record.get("actual_high"), + "deb_prediction": record.get("deb_prediction"), + "mu": record.get("mu"), + "prob_snapshot": record.get("prob_snapshot"), + } + ) + rows.sort(key=lambda row: (row["target_date"], row["city"])) + return rows + + +def _build_training_accuracy_payload( + history: Dict[str, Dict[str, Dict[str, Any]]], + city_registry: Dict[str, Dict[str, Any]], + *, + today_str: Optional[str] = None, +) -> Dict[str, Any]: + from src.analysis.deb_evaluation import backtest_deb_versions + + today = today_str or datetime.now(timezone.utc).strftime("%Y-%m-%d") + accuracy_data: List[Dict[str, Any]] = [] + for city_id, info in (city_registry or {}).items(): + city_rows = history.get(city_id) or history.get(str(city_id).strip().lower()) or {} + deb_payload = _build_city_deb_accuracy(city_id, city_rows, today) + mu_payload = _build_city_mu_accuracy(city_id, city_rows, today) if deb_payload or mu_payload: accuracy_data.append( - {"city_id": city_id, "name": name, "deb": deb_payload, "mu": mu_payload} + { + "city_id": city_id, + "name": (info or {}).get("name") or city_id, + "deb": deb_payload, + "mu": mu_payload, + } ) - # Sort by total days of DEB or Mu accuracy_data.sort( - key=lambda x: max( - x["deb"]["total_days"] if x["deb"] else 0, - x["mu"]["total_days"] if x["mu"] else 0, + key=lambda row: max( + row["deb"]["total_days"] if row.get("deb") else 0, + row["mu"]["total_days"] if row.get("mu") else 0, ), reverse=True, ) - return {"accuracy": accuracy_data} + all_rows = _flatten_training_history(history, today) + today_date = datetime.strptime(today, "%Y-%m-%d").date() + recent_7_start = (today_date - timedelta(days=7)).isoformat() + recent_14_start = (today_date - timedelta(days=14)).isoformat() + end_date = (today_date - timedelta(days=1)).isoformat() + versions = backtest_deb_versions( + all_rows, + min_train_samples=2, + ).get("versions", {}) + + return { + "accuracy": accuracy_data, + "deb_summary": { + "historical": _build_deb_historical_summary(accuracy_data), + "recent_7d": _evaluate_deb_records( + all_rows, + start_date=recent_7_start, + end_date=end_date, + ), + "recent_14d": _evaluate_deb_records( + all_rows, + start_date=recent_14_start, + end_date=end_date, + ), + "versions": versions, + }, + } + + +def get_ops_training_accuracy(request: Request) -> Dict[str, Any]: + from src.analysis.deb_algorithm import load_history + from src.data_collection.city_registry import CITY_REGISTRY + + history_file = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "data", + "daily_records.json", + ) + history = load_history(history_file) + return _build_training_accuracy_payload(history, CITY_REGISTRY) def get_ops_telegram_audit(request: Request) -> Dict[str, Any]: