Show usable recent DEB accuracy
This commit is contained in:
@@ -40,6 +40,19 @@ type DebHistoricalSummary = {
|
|||||||
hits?: number;
|
hits?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type DebUsableRecentSummary = {
|
||||||
|
window?: "recent_7d" | "recent_14d" | string;
|
||||||
|
city_count?: number;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
hit_rate?: number | null;
|
||||||
|
avg_mae?: number | null;
|
||||||
|
recommendations?: {
|
||||||
|
primary?: number;
|
||||||
|
supporting?: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
type DebVersionSummary = {
|
type DebVersionSummary = {
|
||||||
version?: string;
|
version?: string;
|
||||||
samples?: number;
|
samples?: number;
|
||||||
@@ -51,6 +64,7 @@ type DebVersionSummary = {
|
|||||||
|
|
||||||
type DebSummaryPayload = {
|
type DebSummaryPayload = {
|
||||||
historical?: DebHistoricalSummary;
|
historical?: DebHistoricalSummary;
|
||||||
|
usable_recent?: DebUsableRecentSummary;
|
||||||
recent_7d?: DebWindowSummary;
|
recent_7d?: DebWindowSummary;
|
||||||
recent_14d?: DebWindowSummary;
|
recent_14d?: DebWindowSummary;
|
||||||
versions?: Record<string, DebVersionSummary>;
|
versions?: Record<string, DebVersionSummary>;
|
||||||
@@ -178,6 +192,7 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
|||||||
cities: debSummary?.historical?.city_count ?? debSorted.length,
|
cities: debSummary?.historical?.city_count ?? debSorted.length,
|
||||||
sampleDays: debSummary?.historical?.sample_days,
|
sampleDays: debSummary?.historical?.sample_days,
|
||||||
weightedHit: debSummary?.historical?.weighted_hit_rate,
|
weightedHit: debSummary?.historical?.weighted_hit_rate,
|
||||||
|
usableRecent: debSummary?.usable_recent,
|
||||||
};
|
};
|
||||||
}, [debSorted, debSummary]);
|
}, [debSorted, debSummary]);
|
||||||
|
|
||||||
@@ -218,6 +233,10 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
|||||||
|
|
||||||
const formatPct = (value: number | null | undefined) => value == null ? "--" : `${value.toFixed(1)}%`;
|
const formatPct = (value: number | null | undefined) => value == null ? "--" : `${value.toFixed(1)}%`;
|
||||||
const formatMaybeDeg = (value: number | null | undefined, digits = 1) => value == null ? "--" : `${value.toFixed(digits)}°`;
|
const formatMaybeDeg = (value: number | null | undefined, digits = 1) => value == null ? "--" : `${value.toFixed(digits)}°`;
|
||||||
|
const usableWindowLabel = (window?: string) => {
|
||||||
|
if (window === "recent_14d") return isEn ? "Usable 14d" : "可用近14天";
|
||||||
|
return isEn ? "Usable 7d" : "可用近7天";
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="h-full overflow-auto bg-[#f5f7fa]">
|
<div className="h-full overflow-auto bg-[#f5f7fa]">
|
||||||
@@ -242,7 +261,7 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
|||||||
<div className="grid grid-cols-2 gap-2 mb-3 md:grid-cols-3 xl:grid-cols-6">
|
<div className="grid grid-cols-2 gap-2 mb-3 md:grid-cols-3 xl:grid-cols-6">
|
||||||
{[
|
{[
|
||||||
{ icon: Hash, label: isEn ? "Cities" : "城市数", value: debStats.cities, tone: "blue" },
|
{ icon: Hash, label: isEn ? "Cities" : "城市数", value: debStats.cities, tone: "blue" },
|
||||||
{ icon: Target, label: isEn ? "Historical Avg" : "历史平均", value: `${debStats.avgHit.toFixed(1)}%`, tone: "emerald" },
|
{ icon: Target, label: usableWindowLabel(debStats.usableRecent?.window), value: formatPct(debStats.usableRecent?.hit_rate), tone: "emerald" },
|
||||||
{ icon: TrendingUp, label: isEn ? "Recent 7d" : "近7天", value: formatPct(debSummary?.recent_7d?.hit_rate), tone: "emerald" },
|
{ icon: TrendingUp, label: isEn ? "Recent 7d" : "近7天", value: formatPct(debSummary?.recent_7d?.hit_rate), tone: "emerald" },
|
||||||
{ icon: TrendingUp, label: isEn ? "Recent 14d" : "近14天", value: formatPct(debSummary?.recent_14d?.hit_rate), tone: "purple" },
|
{ icon: TrendingUp, label: isEn ? "Recent 14d" : "近14天", value: formatPct(debSummary?.recent_14d?.hit_rate), tone: "purple" },
|
||||||
{ icon: Thermometer, label: isEn ? "Avg Error" : "平均误差", value: `${debStats.avgMae.toFixed(1)}°`, tone: "amber" },
|
{ icon: Thermometer, label: isEn ? "Avg Error" : "平均误差", value: `${debStats.avgMae.toFixed(1)}°`, tone: "amber" },
|
||||||
|
|||||||
@@ -89,6 +89,18 @@ interface DebSummary {
|
|||||||
sample_days?: number;
|
sample_days?: number;
|
||||||
city_count?: number;
|
city_count?: number;
|
||||||
};
|
};
|
||||||
|
usable_recent?: {
|
||||||
|
window?: string;
|
||||||
|
city_count?: number;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
hit_rate?: number | null;
|
||||||
|
avg_mae?: number | null;
|
||||||
|
recommendations?: {
|
||||||
|
primary?: number;
|
||||||
|
supporting?: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
recent_7d?: {
|
recent_7d?: {
|
||||||
hit_rate?: number | null;
|
hit_rate?: number | null;
|
||||||
mae?: number | null;
|
mae?: number | null;
|
||||||
@@ -160,6 +172,7 @@ export function TrainingPageClient() {
|
|||||||
return {
|
return {
|
||||||
avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit,
|
avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit,
|
||||||
avgMae: debSummary?.historical?.avg_mae ?? avgMae,
|
avgMae: debSummary?.historical?.avg_mae ?? avgMae,
|
||||||
|
usableRecent: debSummary?.usable_recent,
|
||||||
recent7Hit: debSummary?.recent_7d?.hit_rate,
|
recent7Hit: debSummary?.recent_7d?.hit_rate,
|
||||||
recent14Hit: debSummary?.recent_14d?.hit_rate,
|
recent14Hit: debSummary?.recent_14d?.hit_rate,
|
||||||
best,
|
best,
|
||||||
@@ -167,6 +180,9 @@ export function TrainingPageClient() {
|
|||||||
};
|
};
|
||||||
}, [accuracy, debSummary]);
|
}, [accuracy, debSummary]);
|
||||||
|
|
||||||
|
const usableWindowLabel = (window?: string) =>
|
||||||
|
window === "recent_14d" ? "DEB 可用近 14 天命中" : "DEB 可用近 7 天命中";
|
||||||
|
|
||||||
const debChartData = useMemo(() => {
|
const debChartData = useMemo(() => {
|
||||||
if (!accuracy?.length) return [];
|
if (!accuracy?.length) return [];
|
||||||
return accuracy
|
return accuracy
|
||||||
@@ -247,7 +263,9 @@ export function TrainingPageClient() {
|
|||||||
<div className="grid grid-cols-2 md:grid-cols-6 gap-4">
|
<div className="grid grid-cols-2 md:grid-cols-6 gap-4">
|
||||||
<KpiCard
|
<KpiCard
|
||||||
icon={Target} color="bg-cyan-500/20 text-cyan-400"
|
icon={Target} color="bg-cyan-500/20 text-cyan-400"
|
||||||
label="DEB 历史平均命中" value={`${kpis.avgHit.toFixed(1)}%`}
|
label={usableWindowLabel(kpis.usableRecent?.window)}
|
||||||
|
value={kpis.usableRecent?.hit_rate == null ? "—" : `${kpis.usableRecent.hit_rate.toFixed(1)}%`}
|
||||||
|
sub={`可用城市 ${kpis.usableRecent?.city_count ?? 0} · 样本 ${kpis.usableRecent?.samples ?? 0} · 历史 ${kpis.avgHit.toFixed(1)}%`}
|
||||||
/>
|
/>
|
||||||
<KpiCard
|
<KpiCard
|
||||||
icon={Target} color="bg-emerald-500/20 text-emerald-400"
|
icon={Target} color="bg-emerald-500/20 text-emerald-400"
|
||||||
|
|||||||
@@ -202,6 +202,18 @@ export const opsApi = {
|
|||||||
sample_days?: number;
|
sample_days?: number;
|
||||||
hits?: number;
|
hits?: number;
|
||||||
};
|
};
|
||||||
|
usable_recent?: {
|
||||||
|
window?: string;
|
||||||
|
city_count?: number;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
hit_rate?: number | null;
|
||||||
|
avg_mae?: number | null;
|
||||||
|
recommendations?: {
|
||||||
|
primary?: number;
|
||||||
|
supporting?: number;
|
||||||
|
};
|
||||||
|
};
|
||||||
recent_7d?: {
|
recent_7d?: {
|
||||||
start_date?: string | null;
|
start_date?: string | null;
|
||||||
end_date?: string | null;
|
end_date?: string | null;
|
||||||
|
|||||||
@@ -90,6 +90,47 @@ def test_training_accuracy_payload_includes_city_deb_trust_strategy():
|
|||||||
assert rows["gamma"]["deb_recent"]["recommendation"] == "insufficient"
|
assert rows["gamma"]["deb_recent"]["recommendation"] == "insufficient"
|
||||||
|
|
||||||
|
|
||||||
|
def test_training_accuracy_payload_summarizes_usable_recent_deb_cities():
|
||||||
|
history = {
|
||||||
|
"alpha": {
|
||||||
|
"2026-06-01": {"actual_high": 20.0, "deb_prediction": 20.1},
|
||||||
|
"2026-06-02": {"actual_high": 21.0, "deb_prediction": 21.1},
|
||||||
|
"2026-06-03": {"actual_high": 22.0, "deb_prediction": 22.1},
|
||||||
|
"2026-06-04": {"actual_high": 23.0, "deb_prediction": 23.1},
|
||||||
|
},
|
||||||
|
"beta": {
|
||||||
|
"2026-06-01": {"actual_high": 30.0, "deb_prediction": 28.0},
|
||||||
|
"2026-06-02": {"actual_high": 31.0, "deb_prediction": 29.0},
|
||||||
|
"2026-06-03": {"actual_high": 32.0, "deb_prediction": 30.0},
|
||||||
|
"2026-06-04": {"actual_high": 33.0, "deb_prediction": 31.0},
|
||||||
|
},
|
||||||
|
"gamma": {
|
||||||
|
"2026-06-04": {"actual_high": 25.0, "deb_prediction": 25.1},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
registry = {
|
||||||
|
"alpha": {"name": "Alpha"},
|
||||||
|
"beta": {"name": "Beta"},
|
||||||
|
"gamma": {"name": "Gamma"},
|
||||||
|
}
|
||||||
|
|
||||||
|
payload = _build_training_accuracy_payload(
|
||||||
|
history,
|
||||||
|
registry,
|
||||||
|
today_str="2026-06-07",
|
||||||
|
)
|
||||||
|
|
||||||
|
usable = payload["deb_summary"]["usable_recent"]
|
||||||
|
|
||||||
|
assert usable["window"] == "recent_7d"
|
||||||
|
assert usable["city_count"] == 1
|
||||||
|
assert usable["samples"] == 4
|
||||||
|
assert usable["hits"] == 4
|
||||||
|
assert usable["hit_rate"] == 100.0
|
||||||
|
assert usable["avg_mae"] == 0.1
|
||||||
|
assert usable["recommendations"] == {"primary": 1, "supporting": 0}
|
||||||
|
|
||||||
|
|
||||||
def test_training_accuracy_payload_caps_version_backtest_samples(monkeypatch):
|
def test_training_accuracy_payload_caps_version_backtest_samples(monkeypatch):
|
||||||
start = date(2025, 1, 1)
|
start = date(2025, 1, 1)
|
||||||
history = {
|
history = {
|
||||||
|
|||||||
@@ -2471,6 +2471,55 @@ def _build_deb_historical_summary(accuracy_data: List[Dict[str, Any]]) -> Dict[s
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_deb_usable_recent_summary(
|
||||||
|
accuracy_data: List[Dict[str, Any]],
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
usable_rows = []
|
||||||
|
recommendations = {"primary": 0, "supporting": 0}
|
||||||
|
for row in accuracy_data:
|
||||||
|
recent = row.get("deb_recent") if isinstance(row.get("deb_recent"), dict) else None
|
||||||
|
if not recent:
|
||||||
|
continue
|
||||||
|
recommendation = str(recent.get("recommendation") or "").strip().lower()
|
||||||
|
if recommendation not in {"primary", "supporting"}:
|
||||||
|
continue
|
||||||
|
usable_rows.append(row)
|
||||||
|
recommendations[recommendation] += 1
|
||||||
|
|
||||||
|
def build_window(window_key: str) -> Dict[str, Any]:
|
||||||
|
samples = 0
|
||||||
|
hits = 0
|
||||||
|
weighted_mae = 0.0
|
||||||
|
city_count = 0
|
||||||
|
for row in usable_rows:
|
||||||
|
recent = row.get("deb_recent") if isinstance(row.get("deb_recent"), dict) else {}
|
||||||
|
metrics = recent.get(window_key) if isinstance(recent.get(window_key), dict) else {}
|
||||||
|
row_samples = int(metrics.get("samples") or 0)
|
||||||
|
if row_samples <= 0:
|
||||||
|
continue
|
||||||
|
row_hits = int(metrics.get("hits") or 0)
|
||||||
|
row_mae = _sf(metrics.get("mae"))
|
||||||
|
samples += row_samples
|
||||||
|
hits += row_hits
|
||||||
|
city_count += 1
|
||||||
|
if row_mae is not None:
|
||||||
|
weighted_mae += row_mae * row_samples
|
||||||
|
return {
|
||||||
|
"window": window_key,
|
||||||
|
"city_count": city_count,
|
||||||
|
"samples": samples,
|
||||||
|
"hits": hits,
|
||||||
|
"hit_rate": _round_metric((hits / samples * 100) if samples else None, 1),
|
||||||
|
"avg_mae": _round_metric((weighted_mae / samples) if samples else None, 2),
|
||||||
|
"recommendations": dict(recommendations),
|
||||||
|
}
|
||||||
|
|
||||||
|
recent_7d = build_window("recent_7d")
|
||||||
|
if int(recent_7d.get("samples") or 0) > 0:
|
||||||
|
return recent_7d
|
||||||
|
return build_window("recent_14d")
|
||||||
|
|
||||||
|
|
||||||
def _flatten_training_history(history: Dict[str, Dict[str, Dict[str, Any]]], today_str: str) -> List[Dict[str, Any]]:
|
def _flatten_training_history(history: Dict[str, Dict[str, Dict[str, Any]]], today_str: str) -> List[Dict[str, Any]]:
|
||||||
rows: List[Dict[str, Any]] = []
|
rows: List[Dict[str, Any]] = []
|
||||||
for city, city_rows in (history or {}).items():
|
for city, city_rows in (history or {}).items():
|
||||||
@@ -2545,6 +2594,7 @@ def _build_training_accuracy_payload(
|
|||||||
"accuracy": accuracy_data,
|
"accuracy": accuracy_data,
|
||||||
"deb_summary": {
|
"deb_summary": {
|
||||||
"historical": _build_deb_historical_summary(accuracy_data),
|
"historical": _build_deb_historical_summary(accuracy_data),
|
||||||
|
"usable_recent": _build_deb_usable_recent_summary(accuracy_data),
|
||||||
"recent_7d": _evaluate_deb_records(
|
"recent_7d": _evaluate_deb_records(
|
||||||
all_rows,
|
all_rows,
|
||||||
start_date=recent_7_start,
|
start_date=recent_7_start,
|
||||||
|
|||||||
Reference in New Issue
Block a user