Improve DEB calibration and training metrics
This commit is contained in:
@@ -19,6 +19,43 @@ type MetricPayload = {
|
|||||||
brier_score?: number;
|
brier_score?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type DebWindowSummary = {
|
||||||
|
start_date?: string | null;
|
||||||
|
end_date?: string | null;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
hit_rate?: number | null;
|
||||||
|
mae?: number | null;
|
||||||
|
bias?: number | null;
|
||||||
|
city_count?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type DebHistoricalSummary = {
|
||||||
|
city_count?: number;
|
||||||
|
avg_hit_rate?: number | null;
|
||||||
|
weighted_hit_rate?: number | null;
|
||||||
|
avg_mae?: number | null;
|
||||||
|
avg_days_per_city?: number;
|
||||||
|
sample_days?: number;
|
||||||
|
hits?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type DebVersionSummary = {
|
||||||
|
version?: string;
|
||||||
|
samples?: number;
|
||||||
|
mae?: number | null;
|
||||||
|
rmse?: number | null;
|
||||||
|
bias?: number | null;
|
||||||
|
bucket_hit_rate?: number | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
type DebSummaryPayload = {
|
||||||
|
historical?: DebHistoricalSummary;
|
||||||
|
recent_7d?: DebWindowSummary;
|
||||||
|
recent_14d?: DebWindowSummary;
|
||||||
|
versions?: Record<string, DebVersionSummary>;
|
||||||
|
};
|
||||||
|
|
||||||
type TrainingCity = {
|
type TrainingCity = {
|
||||||
city_id: string;
|
city_id: string;
|
||||||
name: string;
|
name: string;
|
||||||
@@ -26,6 +63,11 @@ type TrainingCity = {
|
|||||||
mu?: MetricPayload;
|
mu?: MetricPayload;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type TrainingAccuracyPayload = {
|
||||||
|
accuracy: TrainingCity[];
|
||||||
|
deb_summary?: DebSummaryPayload;
|
||||||
|
};
|
||||||
|
|
||||||
const STAT_CARD_CLASSES: Record<string, string> = {
|
const STAT_CARD_CLASSES: Record<string, string> = {
|
||||||
blue: "bg-blue-50 border-blue-200",
|
blue: "bg-blue-50 border-blue-200",
|
||||||
emerald: "bg-emerald-50 border-emerald-200",
|
emerald: "bg-emerald-50 border-emerald-200",
|
||||||
@@ -48,44 +90,48 @@ function barColor(hr: number) {
|
|||||||
const TRAINING_CACHE_KEY = "polyweather_training_accuracy_v1";
|
const TRAINING_CACHE_KEY = "polyweather_training_accuracy_v1";
|
||||||
const TRAINING_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
const TRAINING_CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
||||||
|
|
||||||
function readTrainingCache(): TrainingCity[] | null {
|
function readTrainingCache(): TrainingAccuracyPayload | null {
|
||||||
try {
|
try {
|
||||||
const raw = localStorage.getItem(TRAINING_CACHE_KEY);
|
const raw = localStorage.getItem(TRAINING_CACHE_KEY);
|
||||||
if (!raw) return null;
|
if (!raw) return null;
|
||||||
const cached = JSON.parse(raw);
|
const cached = JSON.parse(raw);
|
||||||
if (cached.ts && Date.now() - cached.ts < TRAINING_CACHE_TTL_MS && Array.isArray(cached.data)) {
|
if (cached.ts && Date.now() - cached.ts < TRAINING_CACHE_TTL_MS) {
|
||||||
return cached.data;
|
if (Array.isArray(cached.data)) return { accuracy: cached.data };
|
||||||
|
if (cached.data && Array.isArray(cached.data.accuracy)) return cached.data;
|
||||||
}
|
}
|
||||||
} catch { /* ignore */ }
|
} catch { /* ignore */ }
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function writeTrainingCache(data: TrainingCity[]) {
|
function writeTrainingCache(data: TrainingAccuracyPayload) {
|
||||||
try {
|
try {
|
||||||
localStorage.setItem(TRAINING_CACHE_KEY, JSON.stringify({ ts: Date.now(), data }));
|
localStorage.setItem(TRAINING_CACHE_KEY, JSON.stringify({ ts: Date.now(), data }));
|
||||||
} catch { /* ignore */ }
|
} catch { /* ignore */ }
|
||||||
}
|
}
|
||||||
|
|
||||||
export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
||||||
const [data, setData] = useState<TrainingCity[] | null>(() => readTrainingCache());
|
const [payload, setPayload] = useState<TrainingAccuracyPayload | null>(() => readTrainingCache());
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
let cancelled = false;
|
let cancelled = false;
|
||||||
fetch("/api/ops/training/accuracy", { cache: "no-store", headers: { Accept: "application/json" } })
|
fetch("/api/ops/training/accuracy", { cache: "no-store", headers: { Accept: "application/json" } })
|
||||||
.then(async (res) => {
|
.then(async (res) => {
|
||||||
if (!res.ok) return null;
|
if (!res.ok) return null;
|
||||||
return res.json() as Promise<{ accuracy: TrainingCity[] }>;
|
return res.json() as Promise<TrainingAccuracyPayload>;
|
||||||
})
|
})
|
||||||
.then((payload) => {
|
.then((nextPayload) => {
|
||||||
if (cancelled || !payload?.accuracy) return;
|
if (cancelled || !nextPayload?.accuracy) return;
|
||||||
const filtered = payload.accuracy.filter((c) => (c.deb || c.mu) && ((c.deb?.total_days ?? 0) + (c.mu?.total_days ?? 0)) >= 5);
|
const filtered = nextPayload.accuracy.filter((c) => (c.deb || c.mu) && ((c.deb?.total_days ?? 0) + (c.mu?.total_days ?? 0)) >= 5);
|
||||||
setData(filtered);
|
const next = { ...nextPayload, accuracy: filtered };
|
||||||
writeTrainingCache(filtered);
|
setPayload(next);
|
||||||
|
writeTrainingCache(next);
|
||||||
})
|
})
|
||||||
.catch(() => {});
|
.catch(() => {});
|
||||||
return () => { cancelled = true; };
|
return () => { cancelled = true; };
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const data = payload?.accuracy ?? null;
|
||||||
|
const debSummary = payload?.deb_summary;
|
||||||
const debSorted = useMemo(() => (data || []).filter((c) => c.deb).sort((a, b) => (b.deb?.hit_rate ?? 0) - (a.deb?.hit_rate ?? 0)), [data]);
|
const debSorted = useMemo(() => (data || []).filter((c) => c.deb).sort((a, b) => (b.deb?.hit_rate ?? 0) - (a.deb?.hit_rate ?? 0)), [data]);
|
||||||
const muSorted = useMemo(() => (data || []).filter((c) => c.mu).sort((a, b) => (b.mu?.hit_rate ?? 0) - (a.mu?.hit_rate ?? 0)), [data]);
|
const muSorted = useMemo(() => (data || []).filter((c) => c.mu).sort((a, b) => (b.mu?.hit_rate ?? 0) - (a.mu?.hit_rate ?? 0)), [data]);
|
||||||
|
|
||||||
@@ -94,8 +140,15 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
|||||||
const avgHit = debSorted.reduce((s, c) => s + (c.deb?.hit_rate ?? 0), 0) / debSorted.length;
|
const avgHit = debSorted.reduce((s, c) => s + (c.deb?.hit_rate ?? 0), 0) / debSorted.length;
|
||||||
const avgMae = debSorted.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debSorted.length;
|
const avgMae = debSorted.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debSorted.length;
|
||||||
const avgDays = Math.round(debSorted.reduce((s, c) => s + (c.deb?.total_days ?? 0), 0) / Math.max(debSorted.length, 1));
|
const avgDays = Math.round(debSorted.reduce((s, c) => s + (c.deb?.total_days ?? 0), 0) / Math.max(debSorted.length, 1));
|
||||||
return { avgHit, avgMae, avgDays, cities: debSorted.length };
|
return {
|
||||||
}, [debSorted]);
|
avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit,
|
||||||
|
avgMae: debSummary?.historical?.avg_mae ?? avgMae,
|
||||||
|
avgDays: debSummary?.historical?.avg_days_per_city ?? avgDays,
|
||||||
|
cities: debSummary?.historical?.city_count ?? debSorted.length,
|
||||||
|
sampleDays: debSummary?.historical?.sample_days,
|
||||||
|
weightedHit: debSummary?.historical?.weighted_hit_rate,
|
||||||
|
};
|
||||||
|
}, [debSorted, debSummary]);
|
||||||
|
|
||||||
const muStats = useMemo(() => {
|
const muStats = useMemo(() => {
|
||||||
if (!muSorted.length) return null;
|
if (!muSorted.length) return null;
|
||||||
@@ -122,6 +175,17 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
|||||||
() => [...muSorted].sort((a, b) => (a.mu?.brier_score ?? 99) - (b.mu?.brier_score ?? 99)).slice(0, 18).map((c) => ({ name: c.name, value: Number((c.mu?.brier_score ?? 0).toFixed(3)) })),
|
() => [...muSorted].sort((a, b) => (a.mu?.brier_score ?? 99) - (b.mu?.brier_score ?? 99)).slice(0, 18).map((c) => ({ name: c.name, value: Number((c.mu?.brier_score ?? 0).toFixed(3)) })),
|
||||||
[muSorted],
|
[muSorted],
|
||||||
);
|
);
|
||||||
|
const debVersionRows = useMemo(() => {
|
||||||
|
const versions = debSummary?.versions || {};
|
||||||
|
return [
|
||||||
|
{ key: "deb_v1_raw", label: isEn ? "Raw DEB" : "原始 DEB" },
|
||||||
|
{ key: "deb_v1_recent_bias_corrected", label: isEn ? "Mean Bias" : "均值偏差" },
|
||||||
|
{ key: "deb_v2_bucket_calibrated", label: isEn ? "Bucket v2" : "桶校准 v2" },
|
||||||
|
].map(({ key, label }) => ({ key, label, value: versions[key] })).filter((row) => row.value);
|
||||||
|
}, [debSummary?.versions, isEn]);
|
||||||
|
|
||||||
|
const formatPct = (value: number | null | undefined) => value == null ? "--" : `${value.toFixed(1)}%`;
|
||||||
|
const formatMaybeDeg = (value: number | null | undefined, digits = 1) => value == null ? "--" : `${value.toFixed(digits)}°`;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="h-full overflow-auto bg-[#f5f7fa]">
|
<div className="h-full overflow-auto bg-[#f5f7fa]">
|
||||||
@@ -143,12 +207,14 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
|||||||
<Thermometer size={14} className="text-amber-600" />
|
<Thermometer size={14} className="text-amber-600" />
|
||||||
{isEn ? "DEB Temperature Forecast" : "DEB 气温预报"}
|
{isEn ? "DEB Temperature Forecast" : "DEB 气温预报"}
|
||||||
</h2>
|
</h2>
|
||||||
<div className="grid grid-cols-4 gap-2 mb-3">
|
<div className="grid grid-cols-2 gap-2 mb-3 md:grid-cols-3 xl:grid-cols-6">
|
||||||
{[
|
{[
|
||||||
{ icon: Hash, label: isEn ? "Cities" : "城市数", value: debStats.cities, tone: "blue" },
|
{ icon: Hash, label: isEn ? "Cities" : "城市数", value: debStats.cities, tone: "blue" },
|
||||||
{ icon: Target, label: isEn ? "Avg Hit" : "平均命中", value: `${debStats.avgHit.toFixed(1)}%`, tone: "emerald" },
|
{ icon: Target, label: isEn ? "Historical Avg" : "历史平均", value: `${debStats.avgHit.toFixed(1)}%`, tone: "emerald" },
|
||||||
|
{ icon: TrendingUp, label: isEn ? "Recent 7d" : "近7天", value: formatPct(debSummary?.recent_7d?.hit_rate), tone: "emerald" },
|
||||||
|
{ icon: TrendingUp, label: isEn ? "Recent 14d" : "近14天", value: formatPct(debSummary?.recent_14d?.hit_rate), tone: "purple" },
|
||||||
{ icon: Thermometer, label: isEn ? "Avg Error" : "平均误差", value: `${debStats.avgMae.toFixed(1)}°`, tone: "amber" },
|
{ icon: Thermometer, label: isEn ? "Avg Error" : "平均误差", value: `${debStats.avgMae.toFixed(1)}°`, tone: "amber" },
|
||||||
{ icon: TrendingUp, label: isEn ? "Avg Days/City" : "每城平均天数", value: debStats.avgDays.toLocaleString(), tone: "purple" },
|
{ icon: Hash, label: isEn ? "Samples" : "样本天数", value: (debStats.sampleDays ?? debStats.avgDays).toLocaleString(), tone: "blue" },
|
||||||
].map(({ icon: Icon, label, value, tone }) => (
|
].map(({ icon: Icon, label, value, tone }) => (
|
||||||
<div key={label} className={`flex items-center gap-3 rounded-lg border ${STAT_CARD_CLASSES[tone]} p-3`}>
|
<div key={label} className={`flex items-center gap-3 rounded-lg border ${STAT_CARD_CLASSES[tone]} p-3`}>
|
||||||
<Icon size={20} className={STAT_ICON_CLASSES[tone]} />
|
<Icon size={20} className={STAT_ICON_CLASSES[tone]} />
|
||||||
@@ -159,6 +225,25 @@ export function TrainingDashboard({ isEn }: { isEn: boolean }) {
|
|||||||
</div>
|
</div>
|
||||||
))}
|
))}
|
||||||
</div>
|
</div>
|
||||||
|
{debVersionRows.length ? (
|
||||||
|
<div className="mb-3 grid grid-cols-1 gap-2 md:grid-cols-3">
|
||||||
|
{debVersionRows.map((row) => {
|
||||||
|
const bucketRate = row.value?.bucket_hit_rate == null ? null : row.value.bucket_hit_rate * 100;
|
||||||
|
return (
|
||||||
|
<div key={row.key} className="rounded-lg border border-slate-200 bg-white px-3 py-2">
|
||||||
|
<div className="flex items-center justify-between gap-3">
|
||||||
|
<span className="text-[11px] font-black uppercase text-slate-500">{row.label}</span>
|
||||||
|
<span className="font-mono text-sm font-black text-slate-900">{formatPct(bucketRate)}</span>
|
||||||
|
</div>
|
||||||
|
<div className="mt-1 flex items-center justify-between text-[11px] text-slate-500">
|
||||||
|
<span>{isEn ? "MAE" : "误差"} {formatMaybeDeg(row.value?.mae, 2)}</span>
|
||||||
|
<span>{isEn ? "Samples" : "样本"} {row.value?.samples ?? 0}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
<div className="grid grid-cols-2 gap-3 mb-4">
|
<div className="grid grid-cols-2 gap-3 mb-4">
|
||||||
<ChartCard title={isEn ? "Forecast Hit Rate by City" : "预报命中率 by 城市"}>
|
<ChartCard title={isEn ? "Forecast Hit Rate by City" : "预报命中率 by 城市"}>
|
||||||
<ResponsiveContainer width="100%" height="100%">
|
<ResponsiveContainer width="100%" height="100%">
|
||||||
|
|||||||
@@ -65,20 +65,44 @@ interface CityAccuracy {
|
|||||||
} | null;
|
} | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface DebSummary {
|
||||||
|
historical?: {
|
||||||
|
avg_hit_rate?: number | null;
|
||||||
|
weighted_hit_rate?: number | null;
|
||||||
|
avg_mae?: number | null;
|
||||||
|
sample_days?: number;
|
||||||
|
city_count?: number;
|
||||||
|
};
|
||||||
|
recent_7d?: {
|
||||||
|
hit_rate?: number | null;
|
||||||
|
mae?: number | null;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
};
|
||||||
|
recent_14d?: {
|
||||||
|
hit_rate?: number | null;
|
||||||
|
mae?: number | null;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function TrainingPageClient() {
|
export function TrainingPageClient() {
|
||||||
const [loading, setLoading] = useState(true);
|
const [loading, setLoading] = useState(true);
|
||||||
const [status, setStatus] = useState<SystemStatusPayload | null>(null);
|
const [status, setStatus] = useState<SystemStatusPayload | null>(null);
|
||||||
const [accuracy, setAccuracy] = useState<CityAccuracy[] | null>(null);
|
const [accuracy, setAccuracy] = useState<CityAccuracy[] | null>(null);
|
||||||
|
const [debSummary, setDebSummary] = useState<DebSummary | null>(null);
|
||||||
|
|
||||||
const load = async () => {
|
const load = async () => {
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
try {
|
try {
|
||||||
const [s, accData] = await Promise.all([
|
const [s, accData] = await Promise.all([
|
||||||
opsApi.systemStatus() as Promise<SystemStatusPayload>,
|
opsApi.systemStatus() as Promise<SystemStatusPayload>,
|
||||||
opsApi.trainingAccuracy().catch(() => ({ accuracy: [] as CityAccuracy[] })),
|
opsApi.trainingAccuracy().catch(() => ({ accuracy: [] as CityAccuracy[], deb_summary: null })),
|
||||||
]);
|
]);
|
||||||
setStatus(s);
|
setStatus(s);
|
||||||
setAccuracy((accData as { accuracy: CityAccuracy[] }).accuracy ?? []);
|
setAccuracy((accData as { accuracy: CityAccuracy[] }).accuracy ?? []);
|
||||||
|
setDebSummary((accData as { deb_summary?: DebSummary | null }).deb_summary ?? null);
|
||||||
} catch { /* */ }
|
} catch { /* */ }
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
};
|
};
|
||||||
@@ -92,8 +116,15 @@ export function TrainingPageClient() {
|
|||||||
const avgMae = debCities.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debCities.length;
|
const avgMae = debCities.reduce((s, c) => s + (c.deb?.mae ?? 0), 0) / debCities.length;
|
||||||
const best = debCities.reduce((a, b) => ((a.deb?.hit_rate ?? 0) > (b.deb?.hit_rate ?? 0) ? a : b));
|
const best = debCities.reduce((a, b) => ((a.deb?.hit_rate ?? 0) > (b.deb?.hit_rate ?? 0) ? a : b));
|
||||||
const worst = debCities.reduce((a, b) => ((a.deb?.mae ?? 0) > (b.deb?.mae ?? 0) ? a : b));
|
const worst = debCities.reduce((a, b) => ((a.deb?.mae ?? 0) > (b.deb?.mae ?? 0) ? a : b));
|
||||||
return { avgHit, avgMae, best, worst };
|
return {
|
||||||
}, [accuracy]);
|
avgHit: debSummary?.historical?.avg_hit_rate ?? avgHit,
|
||||||
|
avgMae: debSummary?.historical?.avg_mae ?? avgMae,
|
||||||
|
recent7Hit: debSummary?.recent_7d?.hit_rate,
|
||||||
|
recent14Hit: debSummary?.recent_14d?.hit_rate,
|
||||||
|
best,
|
||||||
|
worst,
|
||||||
|
};
|
||||||
|
}, [accuracy, debSummary]);
|
||||||
|
|
||||||
const debChartData = useMemo(() => {
|
const debChartData = useMemo(() => {
|
||||||
if (!accuracy?.length) return [];
|
if (!accuracy?.length) return [];
|
||||||
@@ -172,10 +203,18 @@ export function TrainingPageClient() {
|
|||||||
|
|
||||||
{/* Accuracy KPI row */}
|
{/* Accuracy KPI row */}
|
||||||
{kpis ? (
|
{kpis ? (
|
||||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
|
<div className="grid grid-cols-2 md:grid-cols-6 gap-4">
|
||||||
<KpiCard
|
<KpiCard
|
||||||
icon={Target} color="bg-cyan-500/20 text-cyan-400"
|
icon={Target} color="bg-cyan-500/20 text-cyan-400"
|
||||||
label="DEB 平均命中率" value={`${kpis.avgHit.toFixed(1)}%`}
|
label="DEB 历史平均命中" value={`${kpis.avgHit.toFixed(1)}%`}
|
||||||
|
/>
|
||||||
|
<KpiCard
|
||||||
|
icon={Target} color="bg-emerald-500/20 text-emerald-400"
|
||||||
|
label="近 7 天命中" value={kpis.recent7Hit == null ? "—" : `${kpis.recent7Hit.toFixed(1)}%`}
|
||||||
|
/>
|
||||||
|
<KpiCard
|
||||||
|
icon={Activity} color="bg-violet-500/20 text-violet-400"
|
||||||
|
label="近 14 天命中" value={kpis.recent14Hit == null ? "—" : `${kpis.recent14Hit.toFixed(1)}%`}
|
||||||
/>
|
/>
|
||||||
<KpiCard
|
<KpiCard
|
||||||
icon={Activity} color="bg-blue-500/20 text-blue-400"
|
icon={Activity} color="bg-blue-500/20 text-blue-400"
|
||||||
|
|||||||
@@ -154,6 +154,7 @@ export const opsApi = {
|
|||||||
hit_rate: number;
|
hit_rate: number;
|
||||||
mae: number;
|
mae: number;
|
||||||
total_days: number;
|
total_days: number;
|
||||||
|
hits?: number;
|
||||||
details_str: string;
|
details_str: string;
|
||||||
} | null;
|
} | null;
|
||||||
mu?: {
|
mu?: {
|
||||||
@@ -161,9 +162,49 @@ export const opsApi = {
|
|||||||
hit_rate: number;
|
hit_rate: number;
|
||||||
brier_score: number | null;
|
brier_score: number | null;
|
||||||
total_days: number;
|
total_days: number;
|
||||||
|
hits?: number;
|
||||||
details_str: string;
|
details_str: string;
|
||||||
} | null;
|
} | null;
|
||||||
}>;
|
}>;
|
||||||
|
deb_summary?: {
|
||||||
|
historical?: {
|
||||||
|
city_count?: number;
|
||||||
|
avg_hit_rate?: number | null;
|
||||||
|
weighted_hit_rate?: number | null;
|
||||||
|
avg_mae?: number | null;
|
||||||
|
avg_days_per_city?: number;
|
||||||
|
sample_days?: number;
|
||||||
|
hits?: number;
|
||||||
|
};
|
||||||
|
recent_7d?: {
|
||||||
|
start_date?: string | null;
|
||||||
|
end_date?: string | null;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
hit_rate?: number | null;
|
||||||
|
mae?: number | null;
|
||||||
|
bias?: number | null;
|
||||||
|
city_count?: number;
|
||||||
|
};
|
||||||
|
recent_14d?: {
|
||||||
|
start_date?: string | null;
|
||||||
|
end_date?: string | null;
|
||||||
|
samples?: number;
|
||||||
|
hits?: number;
|
||||||
|
hit_rate?: number | null;
|
||||||
|
mae?: number | null;
|
||||||
|
bias?: number | null;
|
||||||
|
city_count?: number;
|
||||||
|
};
|
||||||
|
versions?: Record<string, {
|
||||||
|
version?: string;
|
||||||
|
samples?: number;
|
||||||
|
mae?: number | null;
|
||||||
|
rmse?: number | null;
|
||||||
|
bias?: number | null;
|
||||||
|
bucket_hit_rate?: number | null;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
}>("/api/ops/training/accuracy");
|
}>("/api/ops/training/accuracy");
|
||||||
},
|
},
|
||||||
telegramAudit() {
|
telegramAudit() {
|
||||||
|
|||||||
@@ -1267,8 +1267,10 @@ def calculate_deb_prediction(
|
|||||||
adjustment when enough settled samples exist.
|
adjustment when enough settled samples exist.
|
||||||
"""
|
"""
|
||||||
from src.analysis.deb_evaluation import (
|
from src.analysis.deb_evaluation import (
|
||||||
|
DEB_BUCKET_CALIBRATED_VERSION,
|
||||||
DEB_RAW_VERSION,
|
DEB_RAW_VERSION,
|
||||||
DEB_RECENT_BIAS_CORRECTED_VERSION,
|
DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||||
|
build_bucket_calibrated_corrector,
|
||||||
build_recent_bias_corrector,
|
build_recent_bias_corrector,
|
||||||
flatten_daily_records,
|
flatten_daily_records,
|
||||||
)
|
)
|
||||||
@@ -1297,6 +1299,14 @@ def calculate_deb_prediction(
|
|||||||
lookback_days=bias_lookback_days,
|
lookback_days=bias_lookback_days,
|
||||||
min_samples=bias_min_samples,
|
min_samples=bias_min_samples,
|
||||||
).apply(city_name, raw_prediction)
|
).apply(city_name, raw_prediction)
|
||||||
|
bucket_corrected = build_bucket_calibrated_corrector(
|
||||||
|
history_rows,
|
||||||
|
lookback_days=bias_lookback_days,
|
||||||
|
min_samples=max(5, int(bias_min_samples or 0)),
|
||||||
|
).apply(city_name, raw_prediction)
|
||||||
|
if int(bucket_corrected.get("samples") or 0) > 0:
|
||||||
|
corrected = bucket_corrected
|
||||||
|
|
||||||
bias_adjustment = float(corrected.get("bias_adjustment") or 0.0)
|
bias_adjustment = float(corrected.get("bias_adjustment") or 0.0)
|
||||||
bias_samples = int(corrected.get("samples") or 0)
|
bias_samples = int(corrected.get("samples") or 0)
|
||||||
if bias_samples <= 0:
|
if bias_samples <= 0:
|
||||||
@@ -1311,14 +1321,19 @@ def calculate_deb_prediction(
|
|||||||
|
|
||||||
next_weights_info = weights_info
|
next_weights_info = weights_info
|
||||||
if abs(bias_adjustment) >= 0.05:
|
if abs(bias_adjustment) >= 0.05:
|
||||||
|
correction_label = (
|
||||||
|
"bucket_calibration"
|
||||||
|
if corrected.get("version") == DEB_BUCKET_CALIBRATED_VERSION
|
||||||
|
else "recent_bias"
|
||||||
|
)
|
||||||
next_weights_info = (
|
next_weights_info = (
|
||||||
f"{weights_info or 'DEB'} | "
|
f"{weights_info or 'DEB'} | "
|
||||||
f"recent_bias({bias_adjustment:+.1f},n={bias_samples})"
|
f"{correction_label}({bias_adjustment:+.1f},n={bias_samples})"
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"prediction": corrected["corrected_prediction"],
|
"prediction": corrected["corrected_prediction"],
|
||||||
"raw_prediction": corrected["raw_prediction"],
|
"raw_prediction": corrected["raw_prediction"],
|
||||||
"version": DEB_RECENT_BIAS_CORRECTED_VERSION,
|
"version": corrected.get("version") or DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||||
"weights_info": next_weights_info,
|
"weights_info": next_weights_info,
|
||||||
"bias_adjustment": bias_adjustment,
|
"bias_adjustment": bias_adjustment,
|
||||||
"bias_samples": bias_samples,
|
"bias_samples": bias_samples,
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ from src.analysis.settlement_rounding import apply_city_settlement
|
|||||||
|
|
||||||
DEB_RAW_VERSION = "deb_v1_raw"
|
DEB_RAW_VERSION = "deb_v1_raw"
|
||||||
DEB_RECENT_BIAS_CORRECTED_VERSION = "deb_v1_recent_bias_corrected"
|
DEB_RECENT_BIAS_CORRECTED_VERSION = "deb_v1_recent_bias_corrected"
|
||||||
|
DEB_BUCKET_CALIBRATED_VERSION = "deb_v2_bucket_calibrated"
|
||||||
DEB_BACKTEST_SCHEMA_VERSION = "deb_backtest_report.v1"
|
DEB_BACKTEST_SCHEMA_VERSION = "deb_backtest_report.v1"
|
||||||
|
|
||||||
|
|
||||||
@@ -108,8 +109,14 @@ class BiasCorrectionResult:
|
|||||||
|
|
||||||
|
|
||||||
class RecentBiasCorrector:
|
class RecentBiasCorrector:
|
||||||
def __init__(self, bias_by_city: dict[str, tuple[float, int]]) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
bias_by_city: dict[str, tuple[float, int]],
|
||||||
|
*,
|
||||||
|
version: str = DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||||
|
) -> None:
|
||||||
self._bias_by_city = bias_by_city
|
self._bias_by_city = bias_by_city
|
||||||
|
self._version = version
|
||||||
|
|
||||||
def apply(self, city: str, raw_prediction: float) -> dict[str, Any]:
|
def apply(self, city: str, raw_prediction: float) -> dict[str, Any]:
|
||||||
city_key = str(city or "").strip().lower()
|
city_key = str(city or "").strip().lower()
|
||||||
@@ -117,7 +124,7 @@ class RecentBiasCorrector:
|
|||||||
bias, samples = self._bias_by_city.get(city_key, (0.0, 0))
|
bias, samples = self._bias_by_city.get(city_key, (0.0, 0))
|
||||||
adjustment = round(bias, 1)
|
adjustment = round(bias, 1)
|
||||||
return BiasCorrectionResult(
|
return BiasCorrectionResult(
|
||||||
version=DEB_RECENT_BIAS_CORRECTED_VERSION,
|
version=self._version,
|
||||||
raw_prediction=round(raw, 1),
|
raw_prediction=round(raw, 1),
|
||||||
corrected_prediction=round(raw + adjustment, 1),
|
corrected_prediction=round(raw + adjustment, 1),
|
||||||
bias_adjustment=adjustment,
|
bias_adjustment=adjustment,
|
||||||
@@ -159,6 +166,71 @@ def build_recent_bias_corrector(
|
|||||||
return RecentBiasCorrector(bias_by_city)
|
return RecentBiasCorrector(bias_by_city)
|
||||||
|
|
||||||
|
|
||||||
|
def build_bucket_calibrated_corrector(
|
||||||
|
history: Iterable[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
lookback_days: int = 30,
|
||||||
|
min_samples: int = 5,
|
||||||
|
max_adjustment: float = 3.0,
|
||||||
|
step: float = 0.1,
|
||||||
|
) -> RecentBiasCorrector:
|
||||||
|
by_city: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for record in history:
|
||||||
|
row = _normalise_record(record)
|
||||||
|
if row is None:
|
||||||
|
continue
|
||||||
|
by_city.setdefault(row["city"], []).append(row)
|
||||||
|
|
||||||
|
adjustment_by_city: dict[str, tuple[float, int]] = {}
|
||||||
|
safe_step = max(abs(float(step or 0.1)), 0.1)
|
||||||
|
max_abs = abs(float(max_adjustment or 0.0))
|
||||||
|
candidate_count = int(round((max_abs * 2) / safe_step)) + 1
|
||||||
|
candidates = [
|
||||||
|
round(-max_abs + idx * safe_step, 1)
|
||||||
|
for idx in range(max(candidate_count, 1))
|
||||||
|
]
|
||||||
|
|
||||||
|
for city, rows in by_city.items():
|
||||||
|
rows.sort(key=lambda row: row["target_date"], reverse=True)
|
||||||
|
recent = rows[: max(int(lookback_days or 0), 1)]
|
||||||
|
if len(recent) < min_samples:
|
||||||
|
continue
|
||||||
|
|
||||||
|
best = None
|
||||||
|
for adjustment in candidates:
|
||||||
|
hits = 0
|
||||||
|
total = 0
|
||||||
|
abs_errors: list[float] = []
|
||||||
|
for row in recent:
|
||||||
|
prediction = row["prediction"] + adjustment
|
||||||
|
actual = row["actual"]
|
||||||
|
try:
|
||||||
|
pred_bucket = apply_city_settlement(city, prediction)
|
||||||
|
actual_bucket = apply_city_settlement(city, actual)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if pred_bucket is None or actual_bucket is None:
|
||||||
|
continue
|
||||||
|
total += 1
|
||||||
|
if pred_bucket == actual_bucket:
|
||||||
|
hits += 1
|
||||||
|
abs_errors.append(abs(prediction - actual))
|
||||||
|
if not total:
|
||||||
|
continue
|
||||||
|
mae = statistics.mean(abs_errors) if abs_errors else float("inf")
|
||||||
|
score = (hits, -mae, -abs(adjustment), adjustment)
|
||||||
|
if best is None or score > best:
|
||||||
|
best = score
|
||||||
|
|
||||||
|
if best is not None:
|
||||||
|
adjustment_by_city[city] = (best[3], len(recent))
|
||||||
|
|
||||||
|
return RecentBiasCorrector(
|
||||||
|
adjustment_by_city,
|
||||||
|
version=DEB_BUCKET_CALIBRATED_VERSION,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def backtest_deb_versions(
|
def backtest_deb_versions(
|
||||||
history: Iterable[dict[str, Any]],
|
history: Iterable[dict[str, Any]],
|
||||||
*,
|
*,
|
||||||
@@ -171,6 +243,7 @@ def backtest_deb_versions(
|
|||||||
report_rows: list[dict[str, Any]] = []
|
report_rows: list[dict[str, Any]] = []
|
||||||
raw_eval_rows: list[dict[str, Any]] = []
|
raw_eval_rows: list[dict[str, Any]] = []
|
||||||
corrected_eval_rows: list[dict[str, Any]] = []
|
corrected_eval_rows: list[dict[str, Any]] = []
|
||||||
|
bucket_eval_rows: list[dict[str, Any]] = []
|
||||||
|
|
||||||
by_city: dict[str, list[dict[str, Any]]] = {}
|
by_city: dict[str, list[dict[str, Any]]] = {}
|
||||||
for row in rows:
|
for row in rows:
|
||||||
@@ -181,9 +254,15 @@ def backtest_deb_versions(
|
|||||||
lookback_days=train_lookback_days,
|
lookback_days=train_lookback_days,
|
||||||
min_samples=min_train_samples,
|
min_samples=min_train_samples,
|
||||||
)
|
)
|
||||||
|
bucket_corrector = build_bucket_calibrated_corrector(
|
||||||
|
previous,
|
||||||
|
lookback_days=train_lookback_days,
|
||||||
|
)
|
||||||
corrected = corrector.apply(row["city"], row["prediction"])
|
corrected = corrector.apply(row["city"], row["prediction"])
|
||||||
|
bucket_corrected = bucket_corrector.apply(row["city"], row["prediction"])
|
||||||
raw_prediction = round(row["prediction"], 1)
|
raw_prediction = round(row["prediction"], 1)
|
||||||
corrected_prediction = corrected["corrected_prediction"]
|
corrected_prediction = corrected["corrected_prediction"]
|
||||||
|
bucket_prediction = bucket_corrected["corrected_prediction"]
|
||||||
|
|
||||||
raw_eval_rows.append(
|
raw_eval_rows.append(
|
||||||
{
|
{
|
||||||
@@ -201,6 +280,15 @@ def backtest_deb_versions(
|
|||||||
"actual": row["actual"],
|
"actual": row["actual"],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
if int(bucket_corrected.get("samples") or 0) > 0:
|
||||||
|
bucket_eval_rows.append(
|
||||||
|
{
|
||||||
|
"city": row["city"],
|
||||||
|
"target_date": row["target_date"],
|
||||||
|
"prediction": bucket_prediction,
|
||||||
|
"actual": row["actual"],
|
||||||
|
}
|
||||||
|
)
|
||||||
report_rows.append(
|
report_rows.append(
|
||||||
{
|
{
|
||||||
"city": row["city"],
|
"city": row["city"],
|
||||||
@@ -217,6 +305,12 @@ def backtest_deb_versions(
|
|||||||
"bias_adjustment": corrected["bias_adjustment"],
|
"bias_adjustment": corrected["bias_adjustment"],
|
||||||
"train_samples": corrected["samples"],
|
"train_samples": corrected["samples"],
|
||||||
},
|
},
|
||||||
|
DEB_BUCKET_CALIBRATED_VERSION: {
|
||||||
|
"prediction": bucket_prediction,
|
||||||
|
"error": round(bucket_prediction - row["actual"], 3),
|
||||||
|
"bias_adjustment": bucket_corrected["bias_adjustment"],
|
||||||
|
"train_samples": bucket_corrected["samples"],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -233,6 +327,10 @@ def backtest_deb_versions(
|
|||||||
corrected_eval_rows,
|
corrected_eval_rows,
|
||||||
version=DEB_RECENT_BIAS_CORRECTED_VERSION,
|
version=DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||||
),
|
),
|
||||||
|
DEB_BUCKET_CALIBRATED_VERSION: evaluate_prediction_records(
|
||||||
|
bucket_eval_rows,
|
||||||
|
version=DEB_BUCKET_CALIBRATED_VERSION,
|
||||||
|
),
|
||||||
},
|
},
|
||||||
"rows": report_rows,
|
"rows": report_rows,
|
||||||
}
|
}
|
||||||
@@ -289,6 +387,10 @@ def write_backtest_report(
|
|||||||
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_error",
|
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_error",
|
||||||
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_bias_adjustment",
|
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_bias_adjustment",
|
||||||
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_train_samples",
|
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_train_samples",
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_prediction",
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_error",
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_bias_adjustment",
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_train_samples",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
writer.writeheader()
|
writer.writeheader()
|
||||||
@@ -296,6 +398,7 @@ def write_backtest_report(
|
|||||||
versions = row.get("versions") or {}
|
versions = row.get("versions") or {}
|
||||||
raw = versions.get(DEB_RAW_VERSION) or {}
|
raw = versions.get(DEB_RAW_VERSION) or {}
|
||||||
corrected = versions.get(DEB_RECENT_BIAS_CORRECTED_VERSION) or {}
|
corrected = versions.get(DEB_RECENT_BIAS_CORRECTED_VERSION) or {}
|
||||||
|
bucket = versions.get(DEB_BUCKET_CALIBRATED_VERSION) or {}
|
||||||
writer.writerow(
|
writer.writerow(
|
||||||
{
|
{
|
||||||
"city": row.get("city"),
|
"city": row.get("city"),
|
||||||
@@ -315,5 +418,17 @@ def write_backtest_report(
|
|||||||
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_train_samples": corrected.get(
|
f"{DEB_RECENT_BIAS_CORRECTED_VERSION}_train_samples": corrected.get(
|
||||||
"train_samples"
|
"train_samples"
|
||||||
),
|
),
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_prediction": bucket.get(
|
||||||
|
"prediction"
|
||||||
|
),
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_error": bucket.get(
|
||||||
|
"error"
|
||||||
|
),
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_bias_adjustment": bucket.get(
|
||||||
|
"bias_adjustment"
|
||||||
|
),
|
||||||
|
f"{DEB_BUCKET_CALIBRATED_VERSION}_train_samples": bucket.get(
|
||||||
|
"train_samples"
|
||||||
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -4,9 +4,11 @@ import sys
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from src.analysis.deb_evaluation import (
|
from src.analysis.deb_evaluation import (
|
||||||
|
DEB_BUCKET_CALIBRATED_VERSION,
|
||||||
DEB_RAW_VERSION,
|
DEB_RAW_VERSION,
|
||||||
DEB_RECENT_BIAS_CORRECTED_VERSION,
|
DEB_RECENT_BIAS_CORRECTED_VERSION,
|
||||||
backtest_deb_versions,
|
backtest_deb_versions,
|
||||||
|
build_bucket_calibrated_corrector,
|
||||||
build_recent_bias_corrector,
|
build_recent_bias_corrector,
|
||||||
evaluate_prediction_records,
|
evaluate_prediction_records,
|
||||||
write_backtest_report,
|
write_backtest_report,
|
||||||
@@ -48,6 +50,25 @@ def test_recent_bias_corrector_uses_signed_error_without_rewriting_raw_deb():
|
|||||||
assert corrected["samples"] == 3
|
assert corrected["samples"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_bucket_calibrated_corrector_optimizes_settlement_bucket_hits():
|
||||||
|
history = [
|
||||||
|
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.4, "actual_high": 21.0},
|
||||||
|
{"city": "ankara", "target_date": "2026-05-21", "deb_prediction": 21.4, "actual_high": 22.0},
|
||||||
|
{"city": "ankara", "target_date": "2026-05-22", "deb_prediction": 22.4, "actual_high": 23.0},
|
||||||
|
{"city": "ankara", "target_date": "2026-05-23", "deb_prediction": 23.4, "actual_high": 24.0},
|
||||||
|
{"city": "ankara", "target_date": "2026-05-24", "deb_prediction": 24.4, "actual_high": 25.0},
|
||||||
|
]
|
||||||
|
|
||||||
|
corrector = build_bucket_calibrated_corrector(history, lookback_days=30, min_samples=5)
|
||||||
|
corrected = corrector.apply("ankara", raw_prediction=25.4)
|
||||||
|
|
||||||
|
assert corrected["version"] == DEB_BUCKET_CALIBRATED_VERSION
|
||||||
|
assert corrected["raw_prediction"] == 25.4
|
||||||
|
assert corrected["corrected_prediction"] == 26.0
|
||||||
|
assert corrected["bias_adjustment"] == 0.6
|
||||||
|
assert corrected["samples"] == 5
|
||||||
|
|
||||||
|
|
||||||
def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions():
|
def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions():
|
||||||
history = [
|
history = [
|
||||||
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.0, "actual_high": 22.0},
|
{"city": "ankara", "target_date": "2026-05-20", "deb_prediction": 20.0, "actual_high": 22.0},
|
||||||
@@ -61,6 +82,7 @@ def test_backtest_deb_versions_compares_raw_and_bias_corrected_versions():
|
|||||||
assert report["schema_version"] == "deb_backtest_report.v1"
|
assert report["schema_version"] == "deb_backtest_report.v1"
|
||||||
assert report["versions"][DEB_RAW_VERSION]["samples"] == 2
|
assert report["versions"][DEB_RAW_VERSION]["samples"] == 2
|
||||||
assert report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["samples"] == 2
|
assert report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["samples"] == 2
|
||||||
|
assert report["versions"][DEB_BUCKET_CALIBRATED_VERSION]["samples"] == 0
|
||||||
assert (
|
assert (
|
||||||
report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["mae"]
|
report["versions"][DEB_RECENT_BIAS_CORRECTED_VERSION]["mae"]
|
||||||
< report["versions"][DEB_RAW_VERSION]["mae"]
|
< report["versions"][DEB_RAW_VERSION]["mae"]
|
||||||
|
|||||||
@@ -128,6 +128,52 @@ def test_calculate_deb_prediction_keeps_raw_and_adds_versioned_bias_correction(m
|
|||||||
assert result["bias_samples"] == 3
|
assert result["bias_samples"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_calculate_deb_prediction_prefers_bucket_calibration_when_enough_samples(monkeypatch):
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"src.analysis.deb_algorithm.load_history",
|
||||||
|
lambda _: {
|
||||||
|
"ankara": {
|
||||||
|
"2026-04-11": {
|
||||||
|
"actual_high": 21.0,
|
||||||
|
"deb_prediction": 20.4,
|
||||||
|
"forecasts": {"ECMWF": 20.4, "GFS": 20.4},
|
||||||
|
},
|
||||||
|
"2026-04-12": {
|
||||||
|
"actual_high": 22.0,
|
||||||
|
"deb_prediction": 21.4,
|
||||||
|
"forecasts": {"ECMWF": 21.4, "GFS": 21.4},
|
||||||
|
},
|
||||||
|
"2026-04-13": {
|
||||||
|
"actual_high": 23.0,
|
||||||
|
"deb_prediction": 22.4,
|
||||||
|
"forecasts": {"ECMWF": 22.4, "GFS": 22.4},
|
||||||
|
},
|
||||||
|
"2026-04-14": {
|
||||||
|
"actual_high": 24.0,
|
||||||
|
"deb_prediction": 23.4,
|
||||||
|
"forecasts": {"ECMWF": 23.4, "GFS": 23.4},
|
||||||
|
},
|
||||||
|
"2026-04-15": {
|
||||||
|
"actual_high": 25.0,
|
||||||
|
"deb_prediction": 24.4,
|
||||||
|
"forecasts": {"ECMWF": 24.4, "GFS": 24.4},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = calculate_deb_prediction(
|
||||||
|
"ankara",
|
||||||
|
{"ECMWF": 25.4, "GFS": 25.4},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["raw_prediction"] == 25.4
|
||||||
|
assert result["prediction"] == 26.0
|
||||||
|
assert result["version"] == "deb_v2_bucket_calibrated"
|
||||||
|
assert result["bias_adjustment"] == 0.6
|
||||||
|
assert result["bias_samples"] == 5
|
||||||
|
|
||||||
|
|
||||||
def test_compute_hourly_model_errors_basic():
|
def test_compute_hourly_model_errors_basic():
|
||||||
from src.analysis.deb_algorithm import compute_hourly_model_errors
|
from src.analysis.deb_algorithm import compute_hourly_model_errors
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from web.services.ops_api import _build_training_accuracy_payload
|
||||||
|
|
||||||
|
|
||||||
|
def test_training_accuracy_payload_includes_recent_deb_summary():
|
||||||
|
history = {
|
||||||
|
"alpha": {
|
||||||
|
"2026-05-25": {"actual_high": 20.0, "deb_prediction": 20.0, "mu": 20.0},
|
||||||
|
"2026-06-01": {"actual_high": 21.0, "deb_prediction": 20.4, "mu": 21.0},
|
||||||
|
"2026-06-02": {"actual_high": 22.0, "deb_prediction": 21.6, "mu": 22.0},
|
||||||
|
},
|
||||||
|
"beta": {
|
||||||
|
"2026-06-03": {"actual_high": 30.0, "deb_prediction": 30.2, "mu": 30.0},
|
||||||
|
"2026-06-04": {"actual_high": 31.0, "deb_prediction": 29.2, "mu": 31.0},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
registry = {
|
||||||
|
"alpha": {"name": "Alpha"},
|
||||||
|
"beta": {"name": "Beta"},
|
||||||
|
"gamma": {"name": "Gamma"},
|
||||||
|
}
|
||||||
|
|
||||||
|
payload = _build_training_accuracy_payload(
|
||||||
|
history,
|
||||||
|
registry,
|
||||||
|
today_str="2026-06-07",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [row["city_id"] for row in payload["accuracy"]] == ["alpha", "beta"]
|
||||||
|
assert payload["deb_summary"]["historical"]["city_count"] == 2
|
||||||
|
assert payload["deb_summary"]["historical"]["sample_days"] == 5
|
||||||
|
assert payload["deb_summary"]["recent_7d"]["start_date"] == "2026-05-31"
|
||||||
|
assert payload["deb_summary"]["recent_7d"]["end_date"] == "2026-06-06"
|
||||||
|
assert payload["deb_summary"]["recent_7d"]["samples"] == 4
|
||||||
|
assert payload["deb_summary"]["recent_7d"]["hits"] == 2
|
||||||
|
assert payload["deb_summary"]["recent_7d"]["hit_rate"] == 50.0
|
||||||
|
assert payload["deb_summary"]["recent_14d"]["samples"] == 5
|
||||||
|
assert "deb_v1_raw" in payload["deb_summary"]["versions"]
|
||||||
|
assert "deb_v2_bucket_calibrated" in payload["deb_summary"]["versions"]
|
||||||
+264
-32
@@ -19,6 +19,19 @@ from web.core import GrantPointsRequest
|
|||||||
import web.routes as legacy_routes
|
import web.routes as legacy_routes
|
||||||
|
|
||||||
|
|
||||||
|
def _sf(value: Any) -> Optional[float]:
|
||||||
|
try:
|
||||||
|
if value is None or value == "":
|
||||||
|
return None
|
||||||
|
return float(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _round_metric(value: Optional[float], digits: int = 1) -> Optional[float]:
|
||||||
|
return None if value is None else round(float(value), digits)
|
||||||
|
|
||||||
|
|
||||||
def _require_ops(request: Request) -> Dict[str, Any] | None:
|
def _require_ops(request: Request) -> Dict[str, Any] | None:
|
||||||
# Ops admins are authenticated via Supabase identity + email whitelist.
|
# Ops admins are authenticated via Supabase identity + email whitelist.
|
||||||
# They do NOT need an active Pro subscription to manage the system.
|
# They do NOT need an active Pro subscription to manage the system.
|
||||||
@@ -2212,52 +2225,271 @@ def get_ops_health_check(request: Request) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def get_ops_training_accuracy(request: Request) -> Dict[str, Any]:
|
def _evaluate_deb_records(
|
||||||
from src.analysis.deb_algorithm import get_deb_accuracy, get_mu_accuracy
|
records: List[Dict[str, Any]],
|
||||||
from src.data_collection.city_registry import CITY_REGISTRY
|
*,
|
||||||
|
start_date: Optional[str] = None,
|
||||||
|
end_date: Optional[str] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
from src.analysis.settlement_rounding import apply_city_settlement
|
||||||
|
|
||||||
accuracy_data = []
|
hits = 0
|
||||||
for city_id, info in CITY_REGISTRY.items():
|
total = 0
|
||||||
name = info.get("name") or city_id
|
errors: List[float] = []
|
||||||
|
signed_errors: List[float] = []
|
||||||
|
cities = set()
|
||||||
|
dates = set()
|
||||||
|
for row in records:
|
||||||
|
target_date = str(row.get("target_date") or "").strip()
|
||||||
|
if start_date and target_date < start_date:
|
||||||
|
continue
|
||||||
|
if end_date and target_date > end_date:
|
||||||
|
continue
|
||||||
|
city = str(row.get("city") or "").strip().lower()
|
||||||
|
prediction = _sf(row.get("deb_prediction"))
|
||||||
|
actual = _sf(row.get("actual_high"))
|
||||||
|
if not city or not target_date or prediction is None or actual is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
pred_bucket = apply_city_settlement(city, prediction)
|
||||||
|
actual_bucket = apply_city_settlement(city, actual)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if pred_bucket is None or actual_bucket is None:
|
||||||
|
continue
|
||||||
|
total += 1
|
||||||
|
if pred_bucket == actual_bucket:
|
||||||
|
hits += 1
|
||||||
|
errors.append(abs(prediction - actual))
|
||||||
|
signed_errors.append(prediction - actual)
|
||||||
|
cities.add(city)
|
||||||
|
dates.add(target_date)
|
||||||
|
|
||||||
# Calculate DEB accuracy
|
return {
|
||||||
deb_acc = get_deb_accuracy(city_id)
|
"start_date": start_date,
|
||||||
deb_payload = None
|
"end_date": end_date,
|
||||||
if deb_acc:
|
"samples": total,
|
||||||
deb_payload = {
|
"hits": hits,
|
||||||
"hit_rate": deb_acc[0],
|
"hit_rate": _round_metric((hits / total * 100) if total else None, 1),
|
||||||
"mae": deb_acc[1],
|
"mae": _round_metric((sum(errors) / len(errors)) if errors else None, 2),
|
||||||
"total_days": deb_acc[2],
|
"bias": _round_metric((sum(signed_errors) / len(signed_errors)) if signed_errors else None, 2),
|
||||||
"details_str": deb_acc[3],
|
"city_count": len(cities),
|
||||||
|
"date_count": len(dates),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_city_deb_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]:
|
||||||
|
rows = []
|
||||||
|
for target_date, record in sorted((city_rows or {}).items()):
|
||||||
|
if target_date >= today_str or not isinstance(record, dict):
|
||||||
|
continue
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"city": city_id,
|
||||||
|
"target_date": target_date,
|
||||||
|
"actual_high": record.get("actual_high"),
|
||||||
|
"deb_prediction": record.get("deb_prediction"),
|
||||||
}
|
}
|
||||||
|
)
|
||||||
|
metrics = _evaluate_deb_records(rows)
|
||||||
|
if not metrics["samples"]:
|
||||||
|
return None
|
||||||
|
total = int(metrics["samples"])
|
||||||
|
hits = int(metrics["hits"])
|
||||||
|
mae = float(metrics["mae"] or 0.0)
|
||||||
|
hit_rate = float(metrics["hit_rate"] or 0.0)
|
||||||
|
return {
|
||||||
|
"hit_rate": hit_rate,
|
||||||
|
"mae": mae,
|
||||||
|
"total_days": total,
|
||||||
|
"hits": hits,
|
||||||
|
"details_str": f"过去{total}天 WU命中 {hits}/{total} ({hit_rate:.0f}%) | MAE: {mae:.1f}°",
|
||||||
|
}
|
||||||
|
|
||||||
# Calculate Mu accuracy
|
|
||||||
mu_acc = get_mu_accuracy(city_id)
|
|
||||||
mu_payload = None
|
|
||||||
if mu_acc:
|
|
||||||
mu_payload = {
|
|
||||||
"mae": mu_acc[0],
|
|
||||||
"hit_rate": mu_acc[1],
|
|
||||||
"brier_score": mu_acc[2],
|
|
||||||
"total_days": mu_acc[3],
|
|
||||||
"details_str": mu_acc[4],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
def _build_city_mu_accuracy(city_id: str, city_rows: Dict[str, Dict[str, Any]], today_str: str) -> Optional[Dict[str, Any]]:
|
||||||
|
from src.analysis.settlement_rounding import apply_city_settlement
|
||||||
|
|
||||||
|
errors: List[float] = []
|
||||||
|
hits = 0
|
||||||
|
total = 0
|
||||||
|
brier_scores: List[float] = []
|
||||||
|
for target_date, record in sorted((city_rows or {}).items()):
|
||||||
|
if target_date >= today_str or not isinstance(record, dict):
|
||||||
|
continue
|
||||||
|
actual = _sf(record.get("actual_high"))
|
||||||
|
mu_value = _sf(record.get("mu"))
|
||||||
|
if actual is None or mu_value is None:
|
||||||
|
continue
|
||||||
|
total += 1
|
||||||
|
errors.append(abs(mu_value - actual))
|
||||||
|
if apply_city_settlement(city_id, mu_value) == apply_city_settlement(city_id, actual):
|
||||||
|
hits += 1
|
||||||
|
prob_snapshot = record.get("prob_snapshot") or []
|
||||||
|
if isinstance(prob_snapshot, list):
|
||||||
|
actual_bucket = apply_city_settlement(city_id, actual)
|
||||||
|
score = 0.0
|
||||||
|
used = False
|
||||||
|
for entry in prob_snapshot:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
predicted_p = _sf(entry.get("p")) or 0.0
|
||||||
|
outcome = 1.0 if entry.get("v") == actual_bucket else 0.0
|
||||||
|
score += (predicted_p - outcome) ** 2
|
||||||
|
used = True
|
||||||
|
if used:
|
||||||
|
brier_scores.append(score)
|
||||||
|
|
||||||
|
if not total:
|
||||||
|
return None
|
||||||
|
mae = sum(errors) / len(errors)
|
||||||
|
hit_rate = hits / total * 100
|
||||||
|
brier = (sum(brier_scores) / len(brier_scores)) if brier_scores else None
|
||||||
|
details_parts = [
|
||||||
|
f"μ准确率: 过去{total}天",
|
||||||
|
f"WU命中 {hits}/{total} ({hit_rate:.0f}%)",
|
||||||
|
f"MAE: {mae:.1f}°",
|
||||||
|
]
|
||||||
|
if brier is not None:
|
||||||
|
details_parts.append(f"Brier: {brier:.3f}")
|
||||||
|
return {
|
||||||
|
"mae": mae,
|
||||||
|
"hit_rate": hit_rate,
|
||||||
|
"brier_score": brier,
|
||||||
|
"total_days": total,
|
||||||
|
"hits": hits,
|
||||||
|
"details_str": " | ".join(details_parts),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_deb_historical_summary(accuracy_data: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||||
|
deb_rows = [row for row in accuracy_data if row.get("deb")]
|
||||||
|
if not deb_rows:
|
||||||
|
return {
|
||||||
|
"city_count": 0,
|
||||||
|
"avg_hit_rate": None,
|
||||||
|
"weighted_hit_rate": None,
|
||||||
|
"avg_mae": None,
|
||||||
|
"avg_days_per_city": 0,
|
||||||
|
"sample_days": 0,
|
||||||
|
"hits": 0,
|
||||||
|
}
|
||||||
|
sample_days = sum(int(row["deb"].get("total_days") or 0) for row in deb_rows)
|
||||||
|
hits = sum(int(row["deb"].get("hits") or 0) for row in deb_rows)
|
||||||
|
return {
|
||||||
|
"city_count": len(deb_rows),
|
||||||
|
"avg_hit_rate": _round_metric(
|
||||||
|
sum(float(row["deb"].get("hit_rate") or 0.0) for row in deb_rows) / len(deb_rows),
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
"weighted_hit_rate": _round_metric((hits / sample_days * 100) if sample_days else None, 1),
|
||||||
|
"avg_mae": _round_metric(
|
||||||
|
sum(float(row["deb"].get("mae") or 0.0) for row in deb_rows) / len(deb_rows),
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
"avg_days_per_city": round(sample_days / len(deb_rows)) if deb_rows else 0,
|
||||||
|
"sample_days": sample_days,
|
||||||
|
"hits": hits,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _flatten_training_history(history: Dict[str, Dict[str, Dict[str, Any]]], today_str: str) -> List[Dict[str, Any]]:
|
||||||
|
rows: List[Dict[str, Any]] = []
|
||||||
|
for city, city_rows in (history or {}).items():
|
||||||
|
if not isinstance(city_rows, dict):
|
||||||
|
continue
|
||||||
|
for target_date, record in city_rows.items():
|
||||||
|
if not isinstance(record, dict):
|
||||||
|
continue
|
||||||
|
target_text = str(target_date or "").strip()
|
||||||
|
if not target_text or target_text >= today_str:
|
||||||
|
continue
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"city": str(city or "").strip().lower(),
|
||||||
|
"target_date": target_text,
|
||||||
|
"actual_high": record.get("actual_high"),
|
||||||
|
"deb_prediction": record.get("deb_prediction"),
|
||||||
|
"mu": record.get("mu"),
|
||||||
|
"prob_snapshot": record.get("prob_snapshot"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
rows.sort(key=lambda row: (row["target_date"], row["city"]))
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _build_training_accuracy_payload(
|
||||||
|
history: Dict[str, Dict[str, Dict[str, Any]]],
|
||||||
|
city_registry: Dict[str, Dict[str, Any]],
|
||||||
|
*,
|
||||||
|
today_str: Optional[str] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
from src.analysis.deb_evaluation import backtest_deb_versions
|
||||||
|
|
||||||
|
today = today_str or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
accuracy_data: List[Dict[str, Any]] = []
|
||||||
|
for city_id, info in (city_registry or {}).items():
|
||||||
|
city_rows = history.get(city_id) or history.get(str(city_id).strip().lower()) or {}
|
||||||
|
deb_payload = _build_city_deb_accuracy(city_id, city_rows, today)
|
||||||
|
mu_payload = _build_city_mu_accuracy(city_id, city_rows, today)
|
||||||
if deb_payload or mu_payload:
|
if deb_payload or mu_payload:
|
||||||
accuracy_data.append(
|
accuracy_data.append(
|
||||||
{"city_id": city_id, "name": name, "deb": deb_payload, "mu": mu_payload}
|
{
|
||||||
|
"city_id": city_id,
|
||||||
|
"name": (info or {}).get("name") or city_id,
|
||||||
|
"deb": deb_payload,
|
||||||
|
"mu": mu_payload,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
# Sort by total days of DEB or Mu
|
|
||||||
accuracy_data.sort(
|
accuracy_data.sort(
|
||||||
key=lambda x: max(
|
key=lambda row: max(
|
||||||
x["deb"]["total_days"] if x["deb"] else 0,
|
row["deb"]["total_days"] if row.get("deb") else 0,
|
||||||
x["mu"]["total_days"] if x["mu"] else 0,
|
row["mu"]["total_days"] if row.get("mu") else 0,
|
||||||
),
|
),
|
||||||
reverse=True,
|
reverse=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
return {"accuracy": accuracy_data}
|
all_rows = _flatten_training_history(history, today)
|
||||||
|
today_date = datetime.strptime(today, "%Y-%m-%d").date()
|
||||||
|
recent_7_start = (today_date - timedelta(days=7)).isoformat()
|
||||||
|
recent_14_start = (today_date - timedelta(days=14)).isoformat()
|
||||||
|
end_date = (today_date - timedelta(days=1)).isoformat()
|
||||||
|
versions = backtest_deb_versions(
|
||||||
|
all_rows,
|
||||||
|
min_train_samples=2,
|
||||||
|
).get("versions", {})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"accuracy": accuracy_data,
|
||||||
|
"deb_summary": {
|
||||||
|
"historical": _build_deb_historical_summary(accuracy_data),
|
||||||
|
"recent_7d": _evaluate_deb_records(
|
||||||
|
all_rows,
|
||||||
|
start_date=recent_7_start,
|
||||||
|
end_date=end_date,
|
||||||
|
),
|
||||||
|
"recent_14d": _evaluate_deb_records(
|
||||||
|
all_rows,
|
||||||
|
start_date=recent_14_start,
|
||||||
|
end_date=end_date,
|
||||||
|
),
|
||||||
|
"versions": versions,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def get_ops_training_accuracy(request: Request) -> Dict[str, Any]:
|
||||||
|
from src.analysis.deb_algorithm import load_history
|
||||||
|
from src.data_collection.city_registry import CITY_REGISTRY
|
||||||
|
|
||||||
|
history_file = os.path.join(
|
||||||
|
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
|
||||||
|
"data",
|
||||||
|
"daily_records.json",
|
||||||
|
)
|
||||||
|
history = load_history(history_file)
|
||||||
|
return _build_training_accuracy_payload(history, CITY_REGISTRY)
|
||||||
|
|
||||||
|
|
||||||
def get_ops_telegram_audit(request: Request) -> Dict[str, Any]:
|
def get_ops_telegram_audit(request: Request) -> Dict[str, Any]:
|
||||||
|
|||||||
Reference in New Issue
Block a user