Improve DEB calibration and training metrics
This commit is contained in:
@@ -19,6 +19,43 @@ type MetricPayload = {
|
||||
brier_score?: number;
|
||||
};
|
||||
|
||||
type DebWindowSummary = {
|
||||
start_date?: string | null;
|
||||
end_date?: string | null;
|
||||
samples?: number;
|
||||
hits?: number;
|
||||
hit_rate?: number | null;
|
||||
mae?: number | null;
|
||||
bias?: number | null;
|
||||
city_count?: number;
|
||||
};
|
||||
|
||||
type DebHistoricalSummary = {
|
||||
city_count?: number;
|
||||
avg_hit_rate?: number | null;
|
||||
weighted_hit_rate?: number | null;
|
||||
avg_mae?: number | null;
|
||||
avg_days_per_city?: number;
|
||||
sample_days?: number;
|
||||
hits?: number;
|
||||
};
|
||||
|
||||
type DebVersionSummary = {
|
||||
version?: string;
|
||||
samples?: number;
|
||||
mae?: number | null;
|
||||
rmse?: number | null;
|
||||
bias?: number | null;
|
||||
bucket_hit_rate?: number | null;
|
||||
};
|
||||
|
||||
type DebSummaryPayload = {
|
||||
historical?: DebHistoricalSummary;
|
||||
recent_7d?: DebWindowSummary;
|
||||
recent_14d?: DebWindowSummary;
|
||||
versions?: Record<string, DebVersionSummary>;
|
||||
};
|
||||
|
||||
type TrainingCity = {
|
||||
city_id: string;
|
||||
name: string;
|
||||
@@ -26,6 +63,11 @@ type TrainingCity = {
|
||||
mu?: MetricPayload;
|
||||
};
|
||||
|
||||
type TrainingAccuracyPayload = {
|
||||
accuracy: TrainingCity[];
|
||||
deb_summary?: DebSummaryPayload;
|
||||
};
|
||||
|
||||
const STAT_CARD_CLASSES: Record<string, string> = {
|
||||
blue: "bg-blue-50 border-blue-200",
|
||||
emerald: "bg-emerald-50 border-emerald-200",
|
||||
@@ -48,44 +90,48 @@ function barColor(hr: number) {
|
||||
const TRAINING_CACHE_KEY = "polyweather_training_accuracy_v1";
|
||||
const TRAINING_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||
|
||||
function readTrainingCache(): TrainingCity[] | null {
|
||||
function readTrainingCache(): TrainingAccuracyPayload | null {
|
||||
try {
|
||||
const raw = localStorage.getItem(TRAINING_CACHE_KEY);
|
||||
if (!raw) return null;
|
||||
const cached = JSON.parse(raw);
|
||||
if (cached.ts && Date.now() - cached.ts < TRAINING_CACHE_TTL_MS && Array.isArray(cached.data)) {
|
||||
return cached.data;
|
||||
if (cached.ts && Date.now() - cached.ts < TRAINING_CACHE_TTL_MS) {
|
||||
if (Array.isArray(cached.data)) return { accuracy: cached.data };
|
||||
if (cached.data && Array.isArray(cached.data.accuracy)) return cached.data;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return null;
|
||||
}
|
||||
|
||||
function writeTrainingCache(data: TrainingCity[]) {
|
||||
function writeTrainingCache(data: TrainingAccuracyPayload) {
|
||||
try {
|
||||
localStorage.setItem(TRAINING_CACHE_KEY, JSON.stringify({ ts: Date.now(), data }));
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
|
||||
export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
||||
const [data, setData] = useState<TrainingCity[] | null>(() => readTrainingCache());
|
||||
const [payload, setPayload] = useState<TrainingAccuracyPayload | null>(() => readTrainingCache());
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
fetch("/api/ops/training/accuracy", { cache: "no-store", headers: { Accept: "application/json" } })
|
||||
.then(async (res) => {
|
||||
if (!res.ok) return null;
|
||||
return res.json() as Promise<{ accuracy: TrainingCity[] }>;
|
||||
return res.json() as Promise<TrainingAccuracyPayload>;
|
||||
})
|
||||
.then((payload) => {
|
||||
if (cancelled || !payload?.accuracy) return;
|
||||
const filtered = payload.accuracy.filter((c) => (c.deb || c.mu) && ((c.deb?.total_days ?? 0) + (c.mu?.total_days ?? 0)) >= 5);
|
||||
setData(filtered);
|
||||
writeTrainingCache(filtered);
|
||||
.then((nextPayload) => {
|
||||
if (cancelled || !nextPayload?.accuracy) return;
|
||||
const filtered = nextPayload.accuracy.filter((c) => (c.deb || c.mu) && ((c.deb?.total_days ?? 0) + (c.mu?.total_days ?? 0)) >= 5);
|
||||
const next = { ...nextPayload, accuracy: filtered };
|
||||
setPayload(next);
|
||||
writeTrainingCache(next);
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => { cancelled = true; };
|
||||
}, []);
|
||||
|
||||
const data = payload?.accuracy ?? null;
|
||||
const debSummary = payload?.deb_summary;
|
||||
const debSorted = useMemo(() => (data || []).filter((c) => c.deb).sort((a, b) => (b.deb?.hit_rate ?? 0) - (a.deb?.hit_rate ?? 0)), [data]);
|
||||
const muSorted = useMemo(() => (data || []).filter((c) => c.mu).sort((a, b) => (b.mu?.hit_rate ?? 0) - (a.mu?.hit_rate ?? 0)), [data]);
|
||||
|
||||
@@ -94,8 +140,15 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
||||
const avgHit = debSorted.reduce((s, c) => s + (c.deb?.hit_rate ?? 0), 0) / debSorted.length;
|
||||
const avgMae = debSorted.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debSorted.length;
|
||||
const avgDays = Math.round(debSorted.reduce((s, c) => s + (c.deb?.total_days ?? 0), 0) / Math.max(debSorted.length, 1));
|
||||
return { avgHit, avgMae, avgDays, cities: debSorted.length };
|
||||
}, [debSorted]);
|
||||
return {
|
||||
avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit,
|
||||
avgMae: debSummary?.historical?.avg_mae ?? avgMae,
|
||||
avgDays: debSummary?.historical?.avg_days_per_city ?? avgDays,
|
||||
cities: debSummary?.historical?.city_count ?? debSorted.length,
|
||||
sampleDays: debSummary?.historical?.sample_days,
|
||||
weightedHit: debSummary?.historical?.weighted_hit_rate,
|
||||
};
|
||||
}, [debSorted, debSummary]);
|
||||
|
||||
const muStats = useMemo(() => {
|
||||
if (!muSorted.length) return null;
|
||||
@@ -122,6 +175,17 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
||||
() => [...muSorted].sort((a, b) => (a.mu?.brier_score ?? 99) - (b.mu?.brier_score ?? 99)).slice(0, 18).map((c) => ({ name: c.name, value: Number((c.mu?.brier_score ?? 0).toFixed(3)) })),
|
||||
[muSorted],
|
||||
);
|
||||
const debVersionRows = useMemo(() => {
|
||||
const versions = debSummary?.versions || {};
|
||||
return [
|
||||
{ key: "deb_v1_raw", label: isEn ? "Raw DEB" : "原始 DEB" },
|
||||
{ key: "deb_v1_recent_bias_corrected", label: isEn ? "Mean Bias" : "均值偏差" },
|
||||
{ key: "deb_v2_bucket_calibrated", label: isEn ? "Bucket v2" : "桶校准 v2" },
|
||||
].map(({ key, label }) => ({ key, label, value: versions[key] })).filter((row) => row.value);
|
||||
}, [debSummary?.versions, isEn]);
|
||||
|
||||
const formatPct = (value: number | null | undefined) => value == null ? "--" : `${value.toFixed(1)}%`;
|
||||
const formatMaybeDeg = (value: number | null | undefined, digits = 1) => value == null ? "--" : `${value.toFixed(digits)}°`;
|
||||
|
||||
return (
|
||||
<div className="h-full overflow-auto bg-[#f5f7fa]">
|
||||
@@ -143,12 +207,14 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
||||
<Thermometer size={14} className="text-amber-600" />
|
||||
{isEn ? "DEB Temperature Forecast" : "DEB 气温预报"}
|
||||
</h2>
|
||||
<div className="grid grid-cols-4 gap-2 mb-3">
|
||||
<div className="grid grid-cols-2 gap-2 mb-3 md:grid-cols-3 xl:grid-cols-6">
|
||||
{[
|
||||
{ icon: Hash, label: isEn ? "Cities" : "城市数", value: debStats.cities, tone: "blue" },
|
||||
{ icon: Target, label: isEn ? "Avg Hit" : "平均命中", value: `${debStats.avgHit.toFixed(1)}%`, tone: "emerald" },
|
||||
{ icon: Target, label: isEn ? "Historical Avg" : "历史平均", value: `${debStats.avgHit.toFixed(1)}%`, tone: "emerald" },
|
||||
{ icon: TrendingUp, label: isEn ? "Recent 7d" : "近7天", value: formatPct(debSummary?.recent_7d?.hit_rate), tone: "emerald" },
|
||||
{ icon: TrendingUp, label: isEn ? "Recent 14d" : "近14天", value: formatPct(debSummary?.recent_14d?.hit_rate), tone: "purple" },
|
||||
{ icon: Thermometer, label: isEn ? "Avg Error" : "平均误差", value: `${debStats.avgMae.toFixed(1)}°`, tone: "amber" },
|
||||
{ icon: TrendingUp, label: isEn ? "Avg Days/City" : "每城平均天数", value: debStats.avgDays.toLocaleString(), tone: "purple" },
|
||||
{ icon: Hash, label: isEn ? "Samples" : "样本天数", value: (debStats.sampleDays ?? debStats.avgDays).toLocaleString(), tone: "blue" },
|
||||
].map(({ icon: Icon, label, value, tone }) => (
|
||||
<div key={label} className={`flex items-center gap-3 rounded-lg border ${STAT_CARD_CLASSES[tone]} p-3`}>
|
||||
<Icon size={20} className={STAT_ICON_CLASSES[tone]} />
|
||||
@@ -159,6 +225,25 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{debVersionRows.length ? (
|
||||
<div className="mb-3 grid grid-cols-1 gap-2 md:grid-cols-3">
|
||||
{debVersionRows.map((row) => {
|
||||
const bucketRate = row.value?.bucket_hit_rate == null ? null : row.value.bucket_hit_rate * 100;
|
||||
return (
|
||||
<div key={row.key} className="rounded-lg border border-slate-200 bg-white px-3 py-2">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<span className="text-[11px] font-black uppercase text-slate-500">{row.label}</span>
|
||||
<span className="font-mono text-sm font-black text-slate-900">{formatPct(bucketRate)}</span>
|
||||
</div>
|
||||
<div className="mt-1 flex items-center justify-between text-[11px] text-slate-500">
|
||||
<span>{isEn ? "MAE" : "误差"} {formatMaybeDeg(row.value?.mae, 2)}</span>
|
||||
<span>{isEn ? "Samples" : "样本"} {row.value?.samples ?? 0}</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
) : null}
|
||||
<div className="grid grid-cols-2 gap-3 mb-4">
|
||||
<ChartCard title={isEn ? "Forecast Hit Rate by City" : "预报命中率 by 城市"}>
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
|
||||
@@ -65,20 +65,44 @@ interface CityAccuracy {
|
||||
} | null;
|
||||
}
|
||||
|
||||
interface DebSummary {
|
||||
historical?: {
|
||||
avg_hit_rate?: number | null;
|
||||
weighted_hit_rate?: number | null;
|
||||
avg_mae?: number | null;
|
||||
sample_days?: number;
|
||||
city_count?: number;
|
||||
};
|
||||
recent_7d?: {
|
||||
hit_rate?: number | null;
|
||||
mae?: number | null;
|
||||
samples?: number;
|
||||
hits?: number;
|
||||
};
|
||||
recent_14d?: {
|
||||
hit_rate?: number | null;
|
||||
mae?: number | null;
|
||||
samples?: number;
|
||||
hits?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export function TrainingPageClient() {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [status, setStatus] = useState<SystemStatusPayload | null>(null);
|
||||
const [accuracy, setAccuracy] = useState<CityAccuracy[] | null>(null);
|
||||
const [debSummary, setDebSummary] = useState<DebSummary | null>(null);
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const [s, accData] = await Promise.all([
|
||||
opsApi.systemStatus() as Promise<SystemStatusPayload>,
|
||||
opsApi.trainingAccuracy().catch(() => ({ accuracy: [] as CityAccuracy[] })),
|
||||
opsApi.trainingAccuracy().catch(() => ({ accuracy: [] as CityAccuracy[], deb_summary: null })),
|
||||
]);
|
||||
setStatus(s);
|
||||
setAccuracy((accData as { accuracy: CityAccuracy[] }).accuracy ?? []);
|
||||
setDebSummary((accData as { deb_summary?: DebSummary | null }).deb_summary ?? null);
|
||||
} catch { /* */ }
|
||||
setLoading(false);
|
||||
};
|
||||
@@ -92,8 +116,15 @@ export function TrainingPageClient() {
|
||||
const avgMae = debCities.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debCities.length;
|
||||
const best = debCities.reduce((a, b) => ((a.deb?.hit_rate ?? 0) > (b.deb?.hit_rate ?? 0) ? a : b));
|
||||
const worst = debCities.reduce((a, b) => ((a.deb?.mae ?? 0) > (b.deb?.mae ?? 0) ? a : b));
|
||||
return { avgHit, avgMae, best, worst };
|
||||
}, [accuracy]);
|
||||
return {
|
||||
avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit,
|
||||
avgMae: debSummary?.historical?.avg_mae ?? avgMae,
|
||||
recent7Hit: debSummary?.recent_7d?.hit_rate,
|
||||
recent14Hit: debSummary?.recent_14d?.hit_rate,
|
||||
best,
|
||||
worst,
|
||||
};
|
||||
}, [accuracy, debSummary]);
|
||||
|
||||
const debChartData = useMemo(() => {
|
||||
if (!accuracy?.length) return [];
|
||||
@@ -172,10 +203,18 @@ export function TrainingPageClient() {
|
||||
|
||||
{/* Accuracy KPI row */}
|
||||
{kpis ? (
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
|
||||
<div className="grid grid-cols-2 md:grid-cols-6 gap-4">
|
||||
<KpiCard
|
||||
icon={Target} color="bg-cyan-500/20 text-cyan-400"
|
||||
label="DEB 平均命中率" value={`${kpis.avgHit.toFixed(1)}%`}
|
||||
label="DEB 历史平均命中" value={`${kpis.avgHit.toFixed(1)}%`}
|
||||
/>
|
||||
<KpiCard
|
||||
icon={Target} color="bg-emerald-500/20 text-emerald-400"
|
||||
label="近 7 天命中" value={kpis.recent7Hit == null ? "—" : `${kpis.recent7Hit.toFixed(1)}%`}
|
||||
/>
|
||||
<KpiCard
|
||||
icon={Activity} color="bg-violet-500/20 text-violet-400"
|
||||
label="近 14 天命中" value={kpis.recent14Hit == null ? "—" : `${kpis.recent14Hit.toFixed(1)}%`}
|
||||
/>
|
||||
<KpiCard
|
||||
icon={Activity} color="bg-blue-500/20 text-blue-400"
|
||||
|
||||
@@ -154,6 +154,7 @@ export const opsApi = {
|
||||
hit_rate: number;
|
||||
mae: number;
|
||||
total_days: number;
|
||||
hits?: number;
|
||||
details_str: string;
|
||||
} | null;
|
||||
mu?: {
|
||||
@@ -161,9 +162,49 @@ export const opsApi = {
|
||||
hit_rate: number;
|
||||
brier_score: number | null;
|
||||
total_days: number;
|
||||
hits?: number;
|
||||
details_str: string;
|
||||
} | null;
|
||||
}>;
|
||||
deb_summary?: {
|
||||
historical?: {
|
||||
city_count?: number;
|
||||
avg_hit_rate?: number | null;
|
||||
weighted_hit_rate?: number | null;
|
||||
avg_mae?: number | null;
|
||||
avg_days_per_city?: number;
|
||||
sample_days?: number;
|
||||
hits?: number;
|
||||
};
|
||||
recent_7d?: {
|
||||
start_date?: string | null;
|
||||
end_date?: string | null;
|
||||
samples?: number;
|
||||
hits?: number;
|
||||
hit_rate?: number | null;
|
||||
mae?: number | null;
|
||||
bias?: number | null;
|
||||
city_count?: number;
|
||||
};
|
||||
recent_14d?: {
|
||||
start_date?: string | null;
|
||||
end_date?: string | null;
|
||||
samples?: number;
|
||||
hits?: number;
|
||||
hit_rate?: number | null;
|
||||
mae?: number | null;
|
||||
bias?: number | null;
|
||||
city_count?: number;
|
||||
};
|
||||
versions?: Record<string, {
|
||||
version?: string;
|
||||
samples?: number;
|
||||
mae?: number | null;
|
||||
rmse?: number | null;
|
||||
bias?: number | null;
|
||||
bucket_hit_rate?: number | null;
|
||||
}>;
|
||||
};
|
||||
}>("/api/ops/training/accuracy");
|
||||
},
|
||||
telegramAudit() {
|
||||
|
||||
Reference in New Issue
Block a user