feat: implement real-time observation patch normalization and live temperature threshold visualization logic

This commit is contained in:
2569718930@qq.com
2026-05-27 10:17:33 +08:00
parent e6a673e27d
commit 820dabfbf3
23 changed files with 709 additions and 29 deletions
+18
View File
@@ -1,5 +1,23 @@
# Changelog # Changelog
## 1.8.0 - 2026-05-27
### 新增与重构
- **终端大洲区域过滤与分组**:终端重构支持按大洲/区域过滤与分组,添加移动端大洲 Tab 与卡片流响应式布局。
- **巨鲸盯盘面板**:对接 Polymarket Data API `/holders`,按区域展示 Polymarket 成交量最大的城市、温度合约及真实巨鲸持仓数据。
- **气温走势图升级**:使用 Recharts 交互式图表,支持双向概率分布对比柱状图,并在图表底部渲染 Polymarket 市场点击直达链接。
- **日内偏差动态修正**:引入实时偏差修正算法,用实况观测与多模型小时预报的偏差来动态修正 DEB 预报中枢以及 Mu 概率分布,极大提高了预报和校准的精度。
- **多数据源气温监控图表**:引入 `LiveTemperatureThresholdChart` 组件,展示实时跑道观测、DEB 预报中枢、多模型区间及目标阈值。
- **全站中文化与多语言 (i18n)**:全站支持中英文一键切换,硬编码字符串彻底清理并接入翻译词条。
- **机构落地页与鉴权优化**:首页重构为专业的机构落地页,添加了基于中间件的双层终端门控(/terminal 路由和 landing page 登录态感知)。
- **超大组件拆分与解耦**`AccountCenter` 组件彻底重构拆分为多个细粒度 Hook(`useWalletBind``usePaymentFlow``useBilling`),主组件代码缩减 60%,提升可维护性。
- **Telegram 高频推送与内存优化**:机场观测推送重构,限制 LRU 缓存避免内存膨胀,并针对 Bot 动作和 API 接入进行连接复用与速率限制。
### 修复与优化
- **类型异常修复**:修复在 `_in_peak_time_window` 决策卡时间窗口计算中 `last_h``None` 导致 `NoneType` 异常报错的问题。
- **清理冗余类型转换**:移除 `src/utils/telegram_push.py` 中 8 处冗余的 `str()` 显式包装,精简 Python 代码。
## 1.7.0 - 2026-05-23 ## 1.7.0 - 2026-05-23
### 新增能力 ### 新增能力
+1 -1
View File
@@ -214,5 +214,5 @@ curl http://127.0.0.1:8000/api/payments/runtime
## Version ## Version
- Version: `v1.7.0` - Version: `v1.8.0`
- Last Updated: `2026-05-23` - Last Updated: `2026-05-23`
+1 -1
View File
@@ -230,5 +230,5 @@ POLYWEATHER_OPS_ADMIN_EMAILS=yhrsc30@gmail.com
## 当前版本 ## 当前版本
- 版本:`v1.7.0` - 版本:`v1.8.0`
- 文档最后更新:`2026-05-23` - 文档最后更新:`2026-05-23`
+1 -1
View File
@@ -1 +1 @@
1.7.1 1.8.0
+1 -1
View File
@@ -1,4 +1,4 @@
# PolyWeather API 文档(v1.7.0 # PolyWeather API 文档(v1.8.0
最后更新:`2026-04-27` 最后更新:`2026-04-27`
+1 -1
View File
@@ -1,4 +1,4 @@
# Supabase + 登录 + 支付接入说明(v1.7.0 # Supabase + 登录 + 支付接入说明(v1.8.0
最后更新:`2026-03-14` 最后更新:`2026-03-14`
+1 -1
View File
@@ -1,4 +1,4 @@
# 技术债与工程待办(v1.7.0 # 技术债与工程待办(v1.8.0
最后更新:`2026-05-10` 最后更新:`2026-05-10`
+1 -1
View File
@@ -1,4 +1,4 @@
# PolyWeatherCheckout PolygonScan 验证(v1.7.0 # PolyWeatherCheckout PolygonScan 验证(v1.8.0
最后更新:`2026-03-20` 最后更新:`2026-03-20`
@@ -53,6 +53,8 @@ const PEAK_GLOW_BADGE_CLASS = {
cooling: "border-slate-200 bg-slate-100 text-slate-500", cooling: "border-slate-200 bg-slate-100 text-slate-500",
} as const; } as const;
const PROBABILITY_REFRESH_AFTER_PATCH_MS = 60_000;
function peakGlowLabel(state: keyof typeof PEAK_GLOW_PANEL_CLASS, isEn: boolean) { function peakGlowLabel(state: keyof typeof PEAK_GLOW_PANEL_CLASS, isEn: boolean) {
if (state === "watch") return isEn ? "Watch" : "关注"; if (state === "watch") return isEn ? "Watch" : "关注";
if (state === "near_peak") return isEn ? "Near peak" : "接近峰值"; if (state === "near_peak") return isEn ? "Near peak" : "接近峰值";
@@ -125,6 +127,7 @@ export function LiveTemperatureThresholdChart({
const hasLoadedHourlyDetailRef = useRef(false); const hasLoadedHourlyDetailRef = useRef(false);
const lastPatchAtRef = useRef<number>(Date.now()); const lastPatchAtRef = useRef<number>(Date.now());
const lastAppliedPatchRevisionRef = useRef<number>(0); const lastAppliedPatchRevisionRef = useRef<number>(0);
const lastProbabilityRefreshAtRef = useRef<number>(0);
const localDayRolloverFetchDateRef = useRef<string>(""); const localDayRolloverFetchDateRef = useRef<string>("");
const [showRunwayDetails, setShowRunwayDetails] = useState<boolean>(true); const [showRunwayDetails, setShowRunwayDetails] = useState<boolean>(true);
@@ -150,6 +153,7 @@ export function LiveTemperatureThresholdChart({
hasLoadedHourlyDetailRef.current = false; hasLoadedHourlyDetailRef.current = false;
lastPatchAtRef.current = Date.now(); lastPatchAtRef.current = Date.now();
lastAppliedPatchRevisionRef.current = 0; lastAppliedPatchRevisionRef.current = 0;
lastProbabilityRefreshAtRef.current = 0;
localDayRolloverFetchDateRef.current = ""; localDayRolloverFetchDateRef.current = "";
setCurrentCityLocalDate(formatCityLocalDate(row?.tz_offset_seconds)); setCurrentCityLocalDate(formatCityLocalDate(row?.tz_offset_seconds));
}, [city]); }, [city]);
@@ -222,7 +226,33 @@ export function LiveTemperatureThresholdChart({
const tempValue = validNumber(latestPatch.changes.temp); const tempValue = validNumber(latestPatch.changes.temp);
if (tempValue !== null) setLiveTemp(tempValue); if (tempValue !== null) setLiveTemp(tempValue);
setHourly((prev) => mergePatchIntoHourly(prev ?? seedHourlyForecastFromRow(row), latestPatch)); setHourly((prev) => mergePatchIntoHourly(prev ?? seedHourlyForecastFromRow(row), latestPatch));
}, [latestPatch, row]);
const hasObservationChange =
tempValue !== null ||
Array.isArray(latestPatch.changes.runway_points) ||
Boolean(latestPatch.changes.amos);
if (!hasObservationChange || !shouldPollLiveChart({ city, compact, isActive, isMaximized })) return;
const now = Date.now();
if (now - lastProbabilityRefreshAtRef.current < PROBABILITY_REFRESH_AFTER_PATCH_MS) return;
lastProbabilityRefreshAtRef.current = now;
let cancelled = false;
const refreshProbabilityOverlayAfterPatch = () => {
fetchHourlyForecastForCity(city, { ignoreCache: true, resolution: targetResolution })
.then((data) => {
if (cancelled || !data) return;
hasLoadedHourlyDetailRef.current = true;
setHourly(data);
})
.catch(() => {});
};
refreshProbabilityOverlayAfterPatch();
return () => {
cancelled = true;
};
}, [latestPatch, row, city, targetResolution, compact, isActive, isMaximized]);
useEffect(() => { useEffect(() => {
if (!resyncVersion || !city) return; if (!resyncVersion || !city) return;
@@ -285,6 +315,45 @@ export function LiveTemperatureThresholdChart({
}; };
}, [city, compact, isActive, isMaximized, targetResolution]); }, [city, compact, isActive, isMaximized, targetResolution]);
useEffect(() => {
if (!shouldPollLiveChart({ city, compact, isActive, isMaximized })) return;
let cancelled = false;
const refreshForegroundFullDetail = () => {
lastPatchAtRef.current = Date.now();
fetch(`/api/city/${encodeURIComponent(city)}/summary`)
.then((res) => (res.ok ? res.json() : null))
.then((payload) => {
if (cancelled || !payload) return;
const temp = validNumber(payload?.current?.temp);
if (temp !== null) setLiveTemp(temp);
})
.catch(() => {});
fetchHourlyForecastForCity(city, { ignoreCache: true, resolution: targetResolution })
.then((data) => {
if (cancelled || !data) return;
hasLoadedHourlyDetailRef.current = true;
setHourly(data);
})
.catch(() => {});
};
const handleVisibilityChange = () => {
if (document.visibilityState !== "visible") return;
refreshForegroundFullDetail();
};
document.addEventListener("visibilitychange", handleVisibilityChange);
window.addEventListener("focus", refreshForegroundFullDetail);
return () => {
cancelled = true;
document.removeEventListener("visibilitychange", handleVisibilityChange);
window.removeEventListener("focus", refreshForegroundFullDetail);
};
}, [city, compact, isActive, isMaximized, targetResolution]);
useEffect(() => { useEffect(() => {
if (!city || !currentCityLocalDate) return; if (!city || !currentCityLocalDate) return;
const loadedLocalDate = hourly?.localDate || row?.local_date || ""; const loadedLocalDate = hourly?.localDate || row?.local_date || "";
@@ -318,7 +387,7 @@ export function LiveTemperatureThresholdChart({
}, [hourly, currentCityLocalDate, row?.local_date]); }, [hourly, currentCityLocalDate, row?.local_date]);
const chartLocalDate = chartHourly?.localDate || row?.local_date || currentCityLocalDate; const chartLocalDate = chartHourly?.localDate || row?.local_date || currentCityLocalDate;
const { data, series } = useMemo(() => buildFullDayChartData(row, chartHourly, isEn), [row, chartHourly, isEn]); const { data, series, probabilityOverlay } = useMemo(() => buildFullDayChartData(row, chartHourly, isEn), [row, chartHourly, isEn]);
const peakGlow = useMemo(() => getPeakGlowState(row, data, series), [row, data, series]); const peakGlow = useMemo(() => getPeakGlowState(row, data, series), [row, data, series]);
const autoWindowRange = useMemo( const autoWindowRange = useMemo(
@@ -473,10 +542,13 @@ export function LiveTemperatureThresholdChart({
return list.sort((a, b) => a.threshold - b.threshold); return list.sort((a, b) => a.threshold - b.threshold);
}, [row, allRows]); }, [row, allRows]);
const intDegreeTicks = useMemo(() => buildIntDegreeTicks(activeSeries, zoomedData), [activeSeries, zoomedData]); const intDegreeTicks = useMemo(
() => buildIntDegreeTicks(activeSeries, zoomedData, probabilityOverlay),
[activeSeries, zoomedData, probabilityOverlay],
);
const chartDomain = useMemo( const chartDomain = useMemo(
() => buildChartDomain(activeSeries, zoomedData), () => buildChartDomain(activeSeries, zoomedData, probabilityOverlay),
[activeSeries, zoomedData], [activeSeries, zoomedData, probabilityOverlay],
); );
const subtitle = row ? (isEn ? "Live & Forecast" : "实测与预测") : ""; const subtitle = row ? (isEn ? "Live & Forecast" : "实测与预测") : "";
@@ -669,6 +741,7 @@ export function LiveTemperatureThresholdChart({
cityThresholds={cityThresholds} cityThresholds={cityThresholds}
chartSeries={chartSeries} chartSeries={chartSeries}
activeSeries={activeSeries} activeSeries={activeSeries}
probabilityOverlay={probabilityOverlay}
zoomedData={zoomedData} zoomedData={zoomedData}
chartDomain={chartDomain} chartDomain={chartDomain}
intDegreeTicks={intDegreeTicks} intDegreeTicks={intDegreeTicks}
@@ -15,7 +15,7 @@ import {
} from "recharts"; } from "recharts";
import type { ScanOpportunityRow } from "@/lib/dashboard-types"; import type { ScanOpportunityRow } from "@/lib/dashboard-types";
import { TemperatureTooltipContent } from "@/components/dashboard/scan-terminal/TemperatureTooltipContent"; import { TemperatureTooltipContent } from "@/components/dashboard/scan-terminal/TemperatureTooltipContent";
import type { EvidenceSeries } from "@/components/dashboard/scan-terminal/temperature-chart-logic"; import type { EvidenceSeries, ProbabilityOverlay } from "@/components/dashboard/scan-terminal/temperature-chart-logic";
type CityThreshold = { type CityThreshold = {
threshold: number; threshold: number;
@@ -32,6 +32,7 @@ export function TemperatureChartCanvas({
cityThresholds, cityThresholds,
chartSeries, chartSeries,
activeSeries, activeSeries,
probabilityOverlay,
zoomedData, zoomedData,
chartDomain, chartDomain,
intDegreeTicks, intDegreeTicks,
@@ -55,6 +56,7 @@ export function TemperatureChartCanvas({
cityThresholds: CityThreshold[]; cityThresholds: CityThreshold[];
chartSeries: EvidenceSeries[]; chartSeries: EvidenceSeries[];
activeSeries: EvidenceSeries[]; activeSeries: EvidenceSeries[];
probabilityOverlay: ProbabilityOverlay | null;
zoomedData: Array<Record<string, any>>; zoomedData: Array<Record<string, any>>;
chartDomain: [number, number] | ["auto", "auto"]; chartDomain: [number, number] | ["auto", "auto"];
intDegreeTicks: number[] | null; intDegreeTicks: number[] | null;
@@ -160,6 +162,30 @@ export function TemperatureChartCanvas({
<span>{isEn ? "Show Runway Details" : "显示跑道明细"}</span> <span>{isEn ? "Show Runway Details" : "显示跑道明细"}</span>
</label> </label>
)} )}
{probabilityOverlay && (
<span
className={clsx(
"inline-flex items-center gap-1.5 rounded border border-violet-200 bg-violet-50 px-1.5 py-0.5 text-[10px] font-bold text-violet-700",
canToggleRunwayDetails ? "" : "ml-auto",
)}
title={
probabilityOverlay.muLine
? probabilityOverlay.muLine.label
: isEn
? "Legacy Gaussian probability bands"
: "Legacy 高斯概率温度带"
}
>
<span className="h-2 w-2 rounded-full bg-violet-500/70" />
<span>{isEn ? "Gaussian" : "高斯概率"}</span>
{probabilityOverlay.muLine && (
<span className="font-mono text-violet-600">
μ {probabilityOverlay.muLine.value.toFixed(1)}{tempSymbol}
</span>
)}
</span>
)}
</div> </div>
<div ref={chartHostRef} className="relative min-h-[220px] flex-1"> <div ref={chartHostRef} className="relative min-h-[220px] flex-1">
{canRenderChart && ( {canRenderChart && (
@@ -191,6 +217,16 @@ export function TemperatureChartCanvas({
domain={chartDomain} domain={chartDomain}
ticks={intDegreeTicks ?? undefined} ticks={intDegreeTicks ?? undefined}
/> />
{timeframe === "1D" && probabilityOverlay?.bands.map((band) => (
<ReferenceArea
key={band.key}
y1={band.lower}
y2={band.upper}
strokeOpacity={0}
fill="#8b5cf6"
fillOpacity={band.opacity}
/>
))}
{timeframe === "1D" && cityThresholds.map((t, idx) => { {timeframe === "1D" && cityThresholds.map((t, idx) => {
const isSelected = row && (Number(row.target_threshold ?? row.target_value) === t.threshold); const isSelected = row && (Number(row.target_threshold ?? row.target_value) === t.threshold);
const labelText = isEn const labelText = isEn
@@ -213,6 +249,20 @@ export function TemperatureChartCanvas({
/> />
); );
})} })}
{timeframe === "1D" && probabilityOverlay?.muLine && (
<ReferenceLine
y={probabilityOverlay.muLine.value}
stroke="#7c3aed"
strokeDasharray="2 3"
strokeWidth={1.4}
label={{
value: compact ? undefined : probabilityOverlay.muLine.label,
fill: "#7c3aed",
fontSize: 9,
position: "insideTopLeft",
}}
/>
)}
<Tooltip <Tooltip
filterNull={false} filterNull={false}
cursor={{ stroke: "#94a3b8", strokeWidth: 1 }} cursor={{ stroke: "#94a3b8", strokeWidth: 1 }}
@@ -152,6 +152,19 @@ export function runTests() {
!resyncBlock.includes("setIsHourlyLoading(true)"), !resyncBlock.includes("setIsHourlyLoading(true)"),
"SSE replay resync should refresh full detail in the background without showing the loading overlay", "SSE replay resync should refresh full detail in the background without showing the loading overlay",
); );
assert(
chart.includes("visibilitychange") &&
chart.includes('document.visibilityState !== "visible"') &&
chart.includes("refreshForegroundFullDetail"),
"temperature chart must immediately refresh visible charts when the browser tab returns to the foreground",
);
const foregroundRefreshBlock = chart.match(/const refreshForegroundFullDetail = \(\) => \{[\s\S]*?\n \};/)?.[0] || "";
assert(
foregroundRefreshBlock.includes("ignoreCache: true") &&
foregroundRefreshBlock.includes("fetchHourlyForecastForCity") &&
!foregroundRefreshBlock.includes("setIsHourlyLoading(true)"),
"foreground resume refresh should update full detail immediately in the background without showing the loading overlay",
);
assert(chart.includes("viewMode"), "temperature chart must expose a view mode for DEB-peak auto view versus full-day view"); assert(chart.includes("viewMode"), "temperature chart must expose a view mode for DEB-peak auto view versus full-day view");
assert(chart.includes('useState<"auto" | "full">("full")'), "temperature chart must default every city panel to the all-day view"); assert(chart.includes('useState<"auto" | "full">("full")'), "temperature chart must default every city panel to the all-day view");
assert( assert(
@@ -176,6 +189,19 @@ export function runTests() {
chart.includes("prefersHighFrequencyRunwayResolution") && chart.includes('return "1m";'), chart.includes("prefersHighFrequencyRunwayResolution") && chart.includes('return "1m";'),
"runway charts must request 1-minute detail resolution so historical runway lines match live SSE patch cadence", "runway charts must request 1-minute detail resolution so historical runway lines match live SSE patch cadence",
); );
assert(
chart.includes("PROBABILITY_REFRESH_AFTER_PATCH_MS") &&
chart.includes("lastProbabilityRefreshAtRef") &&
chart.includes("refreshProbabilityOverlayAfterPatch"),
"temperature chart must trigger a throttled background probability refresh after live observation patches",
);
const patchEffectBlock = chart.match(/useEffect\(\(\) => \{\s*if \(!latestPatch[\s\S]*?\}, \[latestPatch, row, city, targetResolution, compact, isActive, isMaximized\]\);/)?.[0] || "";
assert(
patchEffectBlock.includes("refreshProbabilityOverlayAfterPatch") &&
patchEffectBlock.includes("ignoreCache: true") &&
!patchEffectBlock.includes("setIsHourlyLoading(true)"),
"live patch probability refresh must recompute legacy Gaussian in the background without showing a loading overlay",
);
assert(!chartCanvas.includes("ResponsiveContainer"), "temperature chart canvas must not mount Recharts through ResponsiveContainer at 0x0"); assert(!chartCanvas.includes("ResponsiveContainer"), "temperature chart canvas must not mount Recharts through ResponsiveContainer at 0x0");
assert(chartCanvas.includes("ResizeObserver"), "temperature chart canvas must measure its host with ResizeObserver"); assert(chartCanvas.includes("ResizeObserver"), "temperature chart canvas must measure its host with ResizeObserver");
assert( assert(
@@ -10,7 +10,7 @@ import {
__mergePatchIntoHourlyForTest, __mergePatchIntoHourlyForTest,
} from "@/components/dashboard/scan-terminal/LiveTemperatureThresholdChart"; } from "@/components/dashboard/scan-terminal/LiveTemperatureThresholdChart";
function assert(condition: unknown, message: string) { function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message); if (!condition) throw new Error(message);
} }
@@ -802,6 +802,52 @@ export function runTests() {
"latest airport/METAR report should be appended to the live chart series even when history stops earlier", "latest airport/METAR report should be appended to the live chart series even when history stops earlier",
); );
const torontoCanonicalPatchHourly = __mergePatchIntoHourlyForTest(
{
localTime: "19:15",
localDate: "2026-05-27",
times: ["10:00", "13:00", "16:00", "19:00"],
temps: [23, 26, 27, 26],
airportPrimaryTodayObs: [],
} as any,
{
type: "city_observation_patch.v1",
city: "toronto",
revision: 13,
changes: {
temp: 26,
source: "metar",
observed_at_utc: "2026-05-27T23:16:00Z",
observed_at_local: "2026-05-27T19:16:00-04:00",
city_local_date: "2026-05-27",
city_timezone: "America/Toronto",
},
} as any,
);
assert(
torontoCanonicalPatchHourly,
"v1 canonical patch should merge into hourly forecast",
);
const torontoCanonicalPatchChart = __buildTemperatureChartDataForTest(
{
city: "toronto",
local_date: "2026-05-27",
local_time: "19:16",
tz_offset_seconds: -4 * 60 * 60,
temp_symbol: "°C",
} as any,
torontoCanonicalPatchHourly as any,
"1D",
);
assert(
torontoCanonicalPatchHourly.localDate === "2026-05-27",
"v1 canonical patch should update hourly localDate from city_local_date",
);
assert(
torontoCanonicalPatchChart.data.some((point) => point.label === "19:16:00" && point.madis === 26),
"v1 canonical patch observed_at_utc should render at the city-local chart time",
);
const newYorkMinuteStream = __buildTemperatureChartDataForTest( const newYorkMinuteStream = __buildTemperatureChartDataForTest(
{ {
city: "new york", city: "new york",
@@ -1039,4 +1085,45 @@ export function runTests() {
assert(bandPoints.length >= 2, "runway_band tuples should be binned into data slots"); assert(bandPoints.length >= 2, "runway_band tuples should be binned into data slots");
const firstBand = bandPoints[0].runway_band; const firstBand = bandPoints[0].runway_band;
assert(Array.isArray(firstBand) && firstBand[0] === 24.0 && firstBand[1] === 26.0, "runway_band tuple values should match input limits"); assert(Array.isArray(firstBand) && firstBand[0] === 24.0 && firstBand[1] === 26.0, "runway_band tuple values should match input limits");
// ── Legacy Gaussian probability overlay test ──
const gaussianOverlayChart = __buildTemperatureChartDataForTest(
{
city: "toronto",
local_date: "2026-05-27",
local_time: "14:00",
tz_offset_seconds: -4 * 60 * 60,
temp_symbol: "°C",
} as any,
{
localDate: "2026-05-27",
localTime: "14:00",
times: ["10:00", "14:00", "18:00"],
temps: [24, 27, 23],
probabilities: {
mu: 27.4,
engine: "legacy",
distribution_all: [
{ value: 26, probability: 0.18, range: "[25.5~26.5)" },
{ value: 27, probability: 0.42, range: "[26.5~27.5)" },
{ value: 28, probability: 0.31, range: "[27.5~28.5)" },
],
},
} as any,
"1D",
) as any;
const gaussianOverlay = gaussianOverlayChart.probabilityOverlay;
assert(gaussianOverlay, "legacy Gaussian probabilities should be exposed as a chart overlay");
assert(gaussianOverlay.muLine?.value === 27.4, "legacy Gaussian μ should become a reference line");
assert(
gaussianOverlay.bands.some(
(band: any) => band.value === 27 && band.lower === 26.5 && band.upper === 27.5 && band.probability === 0.42,
),
"legacy Gaussian buckets should become horizontal probability temperature bands",
);
assert(
!gaussianOverlayChart.series.some((series: any) => String(series.key || "").includes("probability")),
"legacy Gaussian probability distribution should not be rendered as a time-series line",
);
} }
@@ -5,6 +5,7 @@ import type {
ScanOpportunityRow, ScanOpportunityRow,
ForecastDay, ForecastDay,
DailyModelForecast, DailyModelForecast,
ProbabilityBucket,
} from "@/lib/dashboard-types"; } from "@/lib/dashboard-types";
import { buildDebBaselinePath } from "@/lib/temperature-chart-paths"; import { buildDebBaselinePath } from "@/lib/temperature-chart-paths";
import { DASHBOARD_REFRESH_POLICY_MS } from "@/lib/refresh-policy"; import { DASHBOARD_REFRESH_POLICY_MS } from "@/lib/refresh-policy";
@@ -209,6 +210,35 @@ type EvidenceSeries = {
values: Array<number | null>; values: Array<number | null>;
}; };
type LegacyGaussianProbabilitySource = {
mu?: number | null;
engine?: string | null;
calibration_mode?: string | null;
distribution?: ProbabilityBucket[];
distribution_all?: ProbabilityBucket[];
};
type ProbabilityTemperatureBand = {
key: string;
value: number;
lower: number;
upper: number;
probability: number;
label: string;
opacity: number;
};
type ProbabilityMuLine = {
value: number;
label: string;
};
type ProbabilityOverlay = {
engine: string | null;
muLine: ProbabilityMuLine | null;
bands: ProbabilityTemperatureBand[];
};
type PeakGlowState = "none" | "watch" | "near_peak" | "breakout" | "cooling"; type PeakGlowState = "none" | "watch" | "near_peak" | "breakout" | "cooling";
type PeakGlowMeta = { type PeakGlowMeta = {
@@ -674,6 +704,7 @@ type HourlyForecast = {
airportPrimary?: AirportCurrentConditions | null; airportPrimary?: AirportCurrentConditions | null;
forecastDaily?: ForecastDay[]; forecastDaily?: ForecastDay[];
multiModelDaily?: Record<string, DailyModelForecast>; multiModelDaily?: Record<string, DailyModelForecast>;
probabilities?: LegacyGaussianProbabilitySource | null;
settlementTodayObs?: ObsPoint[]; settlementTodayObs?: ObsPoint[];
settlementStationLabel?: string | null; settlementStationLabel?: string | null;
metarTodayObs?: ObsPoint[]; metarTodayObs?: ObsPoint[];
@@ -697,6 +728,11 @@ function seedHourlyForecastFromRow(row: ScanOpportunityRow | null): HourlyForeca
airportPrimary: null, airportPrimary: null,
forecastDaily: [], forecastDaily: [],
multiModelDaily: {}, multiModelDaily: {},
probabilities: {
engine: row.probability_engine || null,
distribution: row.distribution_preview || [],
distribution_all: row.distribution_full || row.distribution_preview || [],
},
settlementTodayObs: row.settlement_today_obs || row.metar_context?.settlement_today_obs || undefined, settlementTodayObs: row.settlement_today_obs || row.metar_context?.settlement_today_obs || undefined,
metarTodayObs: row.metar_today_obs || row.metar_context?.today_obs || row.metar_recent_obs || row.metar_context?.recent_obs || undefined, metarTodayObs: row.metar_today_obs || row.metar_context?.today_obs || row.metar_recent_obs || row.metar_context?.recent_obs || undefined,
airportPrimaryTodayObs: undefined, airportPrimaryTodayObs: undefined,
@@ -726,6 +762,7 @@ function parseHourlyForecastFromCityDetail(json: CityDetail | null): HourlyForec
airportPrimary: json.airport_primary || null, airportPrimary: json.airport_primary || null,
forecastDaily: json.forecast?.daily || [], forecastDaily: json.forecast?.daily || [],
multiModelDaily: json.multi_model_daily || {}, multiModelDaily: json.multi_model_daily || {},
probabilities: json.probabilities || null,
settlementTodayObs: (json as any).timeseries?.settlement_today_obs || (json as any)?.settlement_today_obs || undefined, settlementTodayObs: (json as any).timeseries?.settlement_today_obs || (json as any)?.settlement_today_obs || undefined,
settlementStationLabel: (json as any)?.settlement_station?.settlement_station_label || null, settlementStationLabel: (json as any)?.settlement_station?.settlement_station_label || null,
metarTodayObs: (json as any).timeseries?.metar_today_obs || (json as any)?.metar_today_obs || undefined, metarTodayObs: (json as any).timeseries?.metar_today_obs || (json as any)?.metar_today_obs || undefined,
@@ -883,7 +920,8 @@ function mergePatchIntoHourly(
): HourlyForecast { ): HourlyForecast {
const changes = patch.changes || {}; const changes = patch.changes || {};
const tempValue = validNumber(changes.temp); const tempValue = validNumber(changes.temp);
const obsTime = typeof changes.obs_time === "string" ? changes.obs_time : null; const observedAtUtc = typeof changes.observed_at_utc === "string" ? changes.observed_at_utc : null;
const obsTime = observedAtUtc || (typeof changes.obs_time === "string" ? changes.obs_time : null);
const source = typeof changes.source === "string" ? changes.source : ""; const source = typeof changes.source === "string" ? changes.source : "";
const explicitHourlyPatch = changes.hourly && typeof changes.hourly === "object" const explicitHourlyPatch = changes.hourly && typeof changes.hourly === "object"
? changes.hourly as Partial<NonNullable<HourlyForecast>> ? changes.hourly as Partial<NonNullable<HourlyForecast>>
@@ -899,6 +937,7 @@ function mergePatchIntoHourly(
temps: [], temps: [],
forecastDaily: [], forecastDaily: [],
multiModelDaily: {}, multiModelDaily: {},
probabilities: null,
}), }),
...explicitHourlyPatch, ...explicitHourlyPatch,
}; };
@@ -906,6 +945,9 @@ function mergePatchIntoHourly(
if (typeof (changes as any).local_date === "string") { if (typeof (changes as any).local_date === "string") {
next.localDate = (changes as any).local_date; next.localDate = (changes as any).local_date;
} }
if (typeof (changes as any).city_local_date === "string") {
next.localDate = (changes as any).city_local_date;
}
if (changes.amos && typeof changes.amos === "object") { if (changes.amos && typeof changes.amos === "object") {
const oldAmos = prev?.amos || {}; const oldAmos = prev?.amos || {};
@@ -976,7 +1018,7 @@ function mergePatchIntoHourly(
if (tempValue !== null) { if (tempValue !== null) {
next.airportCurrent = { next.airportCurrent = {
...(next.airportCurrent || {}), ...(next.airportCurrent || {}),
obs_time: next.airportCurrent?.obs_time ?? null, obs_time: obsTime || next.airportCurrent?.obs_time || null,
temp: tempValue, temp: tempValue,
max_so_far: Math.max( max_so_far: Math.max(
tempValue, tempValue,
@@ -985,7 +1027,7 @@ function mergePatchIntoHourly(
}; };
next.airportPrimary = { next.airportPrimary = {
...(next.airportPrimary || {}), ...(next.airportPrimary || {}),
obs_time: next.airportPrimary?.obs_time ?? null, obs_time: obsTime || next.airportPrimary?.obs_time || null,
temp: tempValue, temp: tempValue,
max_so_far: Math.max( max_so_far: Math.max(
tempValue, tempValue,
@@ -1327,11 +1369,90 @@ function addHourlyTimesToTimeline(
}); });
} }
function probabilityBucketValue(bucket: ProbabilityBucket) {
return validNumber(bucket.value ?? (bucket as any).temp ?? (bucket as any).temperature);
}
function probabilityBucketProbability(bucket: ProbabilityBucket) {
const raw = validNumber(bucket.probability ?? (bucket as any).model_probability);
if (raw === null) return null;
return raw > 1 ? raw / 100 : raw;
}
function probabilityBucketRange(bucket: ProbabilityBucket, value: number) {
const rawRange = String(bucket.range || bucket.bucket || "").trim();
const rangeMatch = rawRange.match(/(-?\d+(?:\.\d+)?)\s*~\s*(-?\d+(?:\.\d+)?)/);
if (rangeMatch) {
const lower = Number(rangeMatch[1]);
const upper = Number(rangeMatch[2]);
if (Number.isFinite(lower) && Number.isFinite(upper) && upper > lower) {
return { lower, upper };
}
}
return {
lower: Number((value - 0.5).toFixed(2)),
upper: Number((value + 0.5).toFixed(2)),
};
}
function buildLegacyGaussianProbabilityOverlay(
row: ScanOpportunityRow | null,
hourly: HourlyForecast,
): ProbabilityOverlay | null {
const source = hourly?.probabilities || null;
const rowBuckets = ((row as any)?.distribution_full || (row as any)?.distribution_preview || []) as ProbabilityBucket[];
const buckets = (
source?.distribution_all?.length
? source.distribution_all
: source?.distribution?.length
? source.distribution
: rowBuckets
) || [];
const engine = source?.engine || row?.probability_engine || (buckets.length ? "legacy" : null);
if (engine && String(engine).toLowerCase() !== "legacy") return null;
const tempSymbol = row?.temp_symbol || "°C";
const bands = buckets
.map((bucket, index) => {
const value = probabilityBucketValue(bucket);
const probability = probabilityBucketProbability(bucket);
if (value === null || probability === null || probability <= 0) return null;
const { lower, upper } = probabilityBucketRange(bucket, value);
return {
key: `legacy_probability_${value}_${index}`,
value,
lower,
upper,
probability,
label: `${value}${tempSymbol} ${Math.round(probability * 100)}%`,
opacity: Number(Math.min(0.16, Math.max(0.035, 0.04 + probability * 0.22)).toFixed(3)),
};
})
.filter((band): band is ProbabilityTemperatureBand => band !== null)
.sort((a, b) => a.value - b.value);
const mu = validNumber(source?.mu);
const muLine = mu === null
? null
: {
value: mu,
label: `Gaussian μ ${mu.toFixed(1)}${tempSymbol}`,
};
if (!bands.length && !muLine) return null;
return {
engine: engine || "legacy",
muLine,
bands,
};
}
function buildFullDayChartData( function buildFullDayChartData(
row: ScanOpportunityRow | null, row: ScanOpportunityRow | null,
hourly: HourlyForecast, hourly: HourlyForecast,
isEn: boolean, isEn: boolean,
): { data: Array<Record<string, any>>; series: EvidenceSeries[] } { ): { data: Array<Record<string, any>>; series: EvidenceSeries[]; probabilityOverlay: ProbabilityOverlay | null } {
const tzOffset = row?.tz_offset_seconds ?? 0; const tzOffset = row?.tz_offset_seconds ?? 0;
const localDateStr = resolveChartLocalDate(row, hourly); const localDateStr = resolveChartLocalDate(row, hourly);
const localDayBounds = getLocalDayBounds(localDateStr); const localDayBounds = getLocalDayBounds(localDateStr);
@@ -1591,7 +1712,9 @@ function buildFullDayChartData(
return point; return point;
}); });
return { data, series }; const probabilityOverlay = buildLegacyGaussianProbabilityOverlay(row, hourly);
return { data, series, probabilityOverlay };
} }
// ── Model summary cards (daily high point predictions) ───────────────── // ── Model summary cards (daily high point predictions) ─────────────────
@@ -1613,13 +1736,27 @@ function buildModelSummaryCards(row: ScanOpportunityRow | null): EvidenceSeries[
// ── Integer-degree ticks for Y-axis ────────────────────────────────── // ── Integer-degree ticks for Y-axis ──────────────────────────────────
function buildIntDegreeTicks(series: EvidenceSeries[], data?: Array<Record<string, string | number | null>>): number[] | null { function probabilityOverlayValues(probabilityOverlay?: ProbabilityOverlay | null) {
if (!probabilityOverlay) return [];
return [
...(probabilityOverlay.muLine ? [probabilityOverlay.muLine.value] : []),
...probabilityOverlay.bands.flatMap((band) => [band.lower, band.upper]),
];
}
function buildIntDegreeTicks(
series: EvidenceSeries[],
data?: Array<Record<string, string | number | null>>,
probabilityOverlay?: ProbabilityOverlay | null,
): number[] | null {
const vals = data?.length const vals = data?.length
? data.flatMap((point) => series.map((s) => point[s.key])).filter((v): v is number => validNumber(v) !== null) ? data.flatMap((point) => series.map((s) => point[s.key])).filter((v): v is number => validNumber(v) !== null)
: series.flatMap((s) => s.values).filter((v): v is number => validNumber(v) !== null); : series.flatMap((s) => s.values).filter((v): v is number => validNumber(v) !== null);
if (!vals.length) return null; const overlayVals = probabilityOverlayValues(probabilityOverlay);
const min = Math.floor(Math.min(...vals)); const allVals = [...vals, ...overlayVals];
const max = Math.ceil(Math.max(...vals)); if (!allVals.length) return null;
const min = Math.floor(Math.min(...allVals));
const max = Math.ceil(Math.max(...allVals));
const ticks: number[] = []; const ticks: number[] = [];
for (let d = min; d <= max; d++) ticks.push(d); for (let d = min; d <= max; d++) ticks.push(d);
return ticks.length > 0 ? ticks : null; return ticks.length > 0 ? ticks : null;
@@ -1628,13 +1765,16 @@ function buildIntDegreeTicks(series: EvidenceSeries[], data?: Array<Record<strin
function buildChartDomain( function buildChartDomain(
series: EvidenceSeries[], series: EvidenceSeries[],
data?: Array<Record<string, string | number | null>>, data?: Array<Record<string, string | number | null>>,
probabilityOverlay?: ProbabilityOverlay | null,
): [number, number] | ["auto", "auto"] { ): [number, number] | ["auto", "auto"] {
const vals = data?.length const vals = data?.length
? data.flatMap((point) => series.map((s) => point[s.key])).filter((v): v is number => validNumber(v) !== null) ? data.flatMap((point) => series.map((s) => point[s.key])).filter((v): v is number => validNumber(v) !== null)
: series.flatMap((s) => s.values).filter((v): v is number => validNumber(v) !== null); : series.flatMap((s) => s.values).filter((v): v is number => validNumber(v) !== null);
if (!vals.length) return ["auto", "auto"]; const overlayVals = probabilityOverlayValues(probabilityOverlay);
const min = Math.min(...vals); const allVals = [...vals, ...overlayVals];
const max = Math.max(...vals); if (!allVals.length) return ["auto", "auto"];
const min = Math.min(...allVals);
const max = Math.max(...allVals);
const span = Math.max(1, max - min); const span = Math.max(1, max - min);
const pad = Math.max(0.5, span * 0.08); const pad = Math.max(0.5, span * 0.08);
return [Number((min - pad).toFixed(1)), Number((max + pad).toFixed(1))]; return [Number((min - pad).toFixed(1)), Number((max + pad).toFixed(1))];
@@ -1913,4 +2053,4 @@ export {
validNumber, validNumber,
}; };
export type { EvidenceSeries, HourlyForecast, PeakGlowMeta, PeakGlowState }; export type { EvidenceSeries, HourlyForecast, PeakGlowMeta, PeakGlowState, ProbabilityOverlay };
+12
View File
@@ -18,6 +18,12 @@ type ObservationPatchV1 = {
city?: string; city?: string;
source?: string; source?: string;
obs_time?: string | null; obs_time?: string | null;
observed_at_utc?: string | null;
observed_at_local?: string | null;
city_local_date?: string | null;
city_timezone?: string | null;
city_utc_offset_seconds?: number | null;
source_cadence_sec?: number | null;
revision?: number; revision?: number;
ts?: number; ts?: number;
payload?: Record<string, unknown>; payload?: Record<string, unknown>;
@@ -210,6 +216,12 @@ function normalizeV1Patch(patch: ObservationPatchV1): CityPatch | null {
...payload, ...payload,
source: typeof patch.source === "string" ? patch.source : payload.source, source: typeof patch.source === "string" ? patch.source : payload.source,
obs_time: typeof patch.obs_time === "string" ? patch.obs_time : payload.obs_time, obs_time: typeof patch.obs_time === "string" ? patch.obs_time : payload.obs_time,
observed_at_utc: typeof patch.observed_at_utc === "string" ? patch.observed_at_utc : payload.observed_at_utc,
observed_at_local: typeof patch.observed_at_local === "string" ? patch.observed_at_local : payload.observed_at_local,
city_local_date: typeof patch.city_local_date === "string" ? patch.city_local_date : payload.city_local_date,
city_timezone: typeof patch.city_timezone === "string" ? patch.city_timezone : payload.city_timezone,
city_utc_offset_seconds: typeof patch.city_utc_offset_seconds === "number" ? patch.city_utc_offset_seconds : payload.city_utc_offset_seconds,
source_cadence_sec: typeof patch.source_cadence_sec === "number" ? patch.source_cadence_sec : payload.source_cadence_sec,
schema_type: V1_EVENT_TYPE, schema_type: V1_EVENT_TYPE,
}; };
+2
View File
@@ -541,6 +541,8 @@ export interface ScanOpportunityRow {
target_unit?: string | null; target_unit?: string | null;
model_probability?: number | null; model_probability?: number | null;
market_probability?: number | null; market_probability?: number | null;
probability_engine?: string | null;
probability_calibration_mode?: string | null;
model_event_probability?: number | null; model_event_probability?: number | null;
raw_model_event_probability?: number | null; raw_model_event_probability?: number | null;
market_event_probability?: number | null; market_event_probability?: number | null;
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "polyweather-frontend", "name": "polyweather-frontend",
"version": "1.7.0", "version": "1.8.0",
"private": true, "private": true,
"scripts": { "scripts": {
"dev": "node scripts/sync-next-server-chunks.mjs --clean-root && next dev", "dev": "node scripts/sync-next-server-chunks.mjs --clean-root && next dev",
+10 -1
View File
@@ -308,7 +308,7 @@ class SettlementSourceMixin:
return None return None
def fetch_cwa_taipei_settlement_current(self) -> Optional[Dict[str, Any]]: def fetch_cwa_taipei_settlement_current(self) -> Optional[Dict[str, Any]]:
cache_key = "cwa:taipei:466920" cache_key = "cwa:466920"
cached = self._get_settlement_cache(cache_key) cached = self._get_settlement_cache(cache_key)
if cached: if cached:
return cached return cached
@@ -365,6 +365,15 @@ class SettlementSourceMixin:
}, },
"unit": "celsius", "unit": "celsius",
} }
today_obs = self._update_official_today_obs(
source_code="cwa",
station_code=payload["station_code"],
obs_iso=payload.get("observation_time"),
current_temp=payload["current"]["temp"],
utc_offset_seconds=28800,
)
if today_obs:
payload["today_obs"] = today_obs
self._set_settlement_cache(cache_key, payload) self._set_settlement_cache(cache_key, payload)
# Write to airport obs log for high-freq monitoring # Write to airport obs log for high-freq monitoring
try: try:
+27
View File
@@ -40,6 +40,33 @@ def test_event_store_appends_monotonic_revisions_and_replays_by_city(tmp_path):
assert replay[0]["payload"]["temp"] == 31.5 assert replay[0]["payload"]["temp"] == 31.5
def test_event_store_preserves_time_contract_on_live_and_replay_events(tmp_path):
db_path = str(tmp_path / "polyweather.db")
DBManager._initialized_paths.clear()
store = RealtimeEventStore(db_path=db_path)
stored = store.append_event(
normalize_observation_patch(
{
"city": "toronto",
"changes": {
"temp": 26,
"obs_time": "2026-05-27T23:16:00Z",
"source": "metar",
},
}
)
)
replayed = store.replay_events(cities={"toronto"}, since_revision=0, limit=10)[0]
assert stored["observed_at_utc"] == "2026-05-27T23:16:00Z"
assert stored["observed_at_local"] == "2026-05-27T19:16:00-04:00"
assert stored["city_local_date"] == "2026-05-27"
assert replayed["observed_at_utc"] == stored["observed_at_utc"]
assert replayed["observed_at_local"] == stored["observed_at_local"]
assert replayed["city_timezone"] == "America/Toronto"
def test_event_store_cleanup_uses_short_replay_retention(tmp_path): def test_event_store_cleanup_uses_short_replay_retention(tmp_path):
db_path = str(tmp_path / "polyweather.db") db_path = str(tmp_path / "polyweather.db")
DBManager._initialized_paths.clear() DBManager._initialized_paths.clear()
+25
View File
@@ -78,6 +78,31 @@ def test_v1_patch_payload_is_accepted_and_normalized():
assert event["payload"]["runway_points"][0]["temp"] == 30.2 assert event["payload"]["runway_points"][0]["temp"] == 30.2
def test_patch_adds_city_local_time_contract_from_observation_time():
event = normalize_observation_patch(
{
"type": "city_observation_patch.v1",
"city": "Toronto",
"source": "metar",
"obs_time": "2026-05-27T23:16:00Z",
"payload": {
"temp": 26,
"station_code": "CYYZ",
},
}
)
assert event["obs_time"] == "2026-05-27T23:16:00Z"
assert event["observed_at_utc"] == "2026-05-27T23:16:00Z"
assert event["observed_at_local"] == "2026-05-27T19:16:00-04:00"
assert event["city_local_date"] == "2026-05-27"
assert event["city_timezone"] == "America/Toronto"
assert event["city_utc_offset_seconds"] == -4 * 60 * 60
assert event["source_cadence_sec"] == 1800
assert event["payload"]["observed_at_utc"] == "2026-05-27T23:16:00Z"
assert event["payload"]["observed_at_local"] == "2026-05-27T19:16:00-04:00"
def test_invalid_patch_without_city_or_observation_data_is_rejected(): def test_invalid_patch_without_city_or_observation_data_is_rejected():
with pytest.raises(PatchValidationError): with pytest.raises(PatchValidationError):
normalize_observation_patch({"changes": {"temp": 21.0}}) normalize_observation_patch({"changes": {"temp": 21.0}})
+98
View File
@@ -0,0 +1,98 @@
import threading
import src.data_collection.settlement_sources as settlement_sources
import src.database.db_manager as db_manager
from src.data_collection.settlement_sources import SettlementSourceMixin
class _FakeResponse:
content = b"{}"
def raise_for_status(self):
return None
def json(self):
return {
"records": {
"Station": [
{
"StationId": "466920",
"StationName": "中央氣象署臺北站",
"ObsTime": {"DateTime": "2026-05-27T10:00:00+08:00"},
"WeatherElement": {
"AirTemperature": "34.2",
"RelativeHumidity": "60",
"WindSpeed": "2.0",
"WindDirection": "90",
"DailyExtreme": {
"DailyHigh": {
"TemperatureInfo": {
"AirTemperature": "34.2",
"Occurred_at": {
"DateTime": "2026-05-27T10:00:00+08:00"
},
}
},
"DailyLow": {
"TemperatureInfo": {"AirTemperature": "27.1"}
},
},
},
}
]
}
}
class _FakeOfficialIntradayRepo:
def __init__(self):
self.rows = [{"time": "09:50", "temp": 34.0}]
self.upserts = []
def upsert_point(self, **kwargs):
self.upserts.append(kwargs)
self.rows.append(
{
"time": kwargs["observation_time"],
"temp": kwargs["value"],
}
)
def load_points(self, **kwargs):
return list(self.rows)
class _FakeDBManager:
def append_airport_obs(self, **kwargs):
return None
class _FakeCollector(SettlementSourceMixin):
cwa_open_data_auth = "token"
timeout = 1
settlement_cache_ttl_sec = 0
def __init__(self):
self._settlement_cache = {}
self._settlement_cache_lock = threading.Lock()
def _http_get(self, *args, **kwargs):
return _FakeResponse()
def test_cwa_settlement_returns_recorded_intraday_history(monkeypatch):
repo = _FakeOfficialIntradayRepo()
monkeypatch.setattr(settlement_sources, "_official_intraday_repo", repo)
monkeypatch.setattr(db_manager, "DBManager", lambda: _FakeDBManager())
collector = _FakeCollector()
payload = collector.fetch_cwa_taipei_settlement_current()
assert payload is not None
assert "cwa:466920" in collector._settlement_cache
assert repo.upserts[0]["source_code"] == "cwa"
assert repo.upserts[0]["station_code"] == "466920"
assert payload["today_obs"] == [
{"time": "09:50", "temp": 34.0},
{"time": "10:00", "temp": 34.2},
]
+14
View File
@@ -14,6 +14,14 @@ from web.realtime_patch_schema import EVENT_TYPE
DEFAULT_RETENTION_HOURS = 6 DEFAULT_RETENTION_HOURS = 6
MAX_REPLAY_LIMIT = 2000 MAX_REPLAY_LIMIT = 2000
TIME_CONTRACT_KEYS = (
"observed_at_utc",
"observed_at_local",
"city_local_date",
"city_timezone",
"city_utc_offset_seconds",
"source_cadence_sec",
)
def _utc_now() -> datetime: def _utc_now() -> datetime:
@@ -43,6 +51,10 @@ def _normalize_city_set(cities: Optional[Set[str]]) -> Set[str]:
return {str(city or "").strip().lower() for city in (cities or set()) if str(city or "").strip()} return {str(city or "").strip().lower() for city in (cities or set()) if str(city or "").strip()}
def _time_contract_from_payload(payload: Dict[str, Any]) -> Dict[str, Any]:
return {key: payload[key] for key in TIME_CONTRACT_KEYS if key in payload}
def _retention_hours_from_env() -> int: def _retention_hours_from_env() -> int:
raw = os.getenv("POLYWEATHER_PATCH_EVENT_RETENTION_HOURS", "").strip() raw = os.getenv("POLYWEATHER_PATCH_EVENT_RETENTION_HOURS", "").strip()
if not raw: if not raw:
@@ -133,6 +145,7 @@ class RealtimeEventStore:
"city": str(event["city"]), "city": str(event["city"]),
"source": str(event["source"]), "source": str(event["source"]),
"obs_time": event.get("obs_time"), "obs_time": event.get("obs_time"),
**_time_contract_from_payload(payload),
"ts": int(event.get("ts") or _created_at_to_ms(created_at)), "ts": int(event.get("ts") or _created_at_to_ms(created_at)),
"payload": payload, "payload": payload,
} }
@@ -246,6 +259,7 @@ class RealtimeEventStore:
"city": str(row["city"]), "city": str(row["city"]),
"source": str(row["source"]), "source": str(row["source"]),
"obs_time": row["obs_time"], "obs_time": row["obs_time"],
**_time_contract_from_payload(payload),
"ts": _created_at_to_ms(row["created_at"]), "ts": _created_at_to_ms(row["created_at"]),
"payload": payload, "payload": payload,
} }
+94
View File
@@ -3,12 +3,36 @@
from __future__ import annotations from __future__ import annotations
import time import time
from datetime import datetime, timedelta, timezone
from typing import Any, Dict, Iterable, List, Optional from typing import Any, Dict, Iterable, List, Optional
from src.data_collection.city_time import (
city_local_datetime,
get_city_timezone_name,
get_city_utc_offset_seconds,
)
SCHEMA_TYPE = "city_observation_patch" SCHEMA_TYPE = "city_observation_patch"
SCHEMA_VERSION = 1 SCHEMA_VERSION = 1
EVENT_TYPE = "city_observation_patch.v1" EVENT_TYPE = "city_observation_patch.v1"
SOURCE_CADENCE_SECONDS = {
"amos": 60,
"amsc_awos": 60,
"cowin_obs": 60,
"hko_obs": 600,
"singapore_mss": 60,
"madis_hfmetar": 300,
"jma_amedas": 600,
"fmi": 600,
"knmi": 600,
"mgm": 300,
"ims": 600,
"ncm": 600,
"aeroweb": 900,
"cwa": 600,
"metar": 1800,
}
class PatchValidationError(ValueError): class PatchValidationError(ValueError):
@@ -42,6 +66,56 @@ def _first_number(*values: Any) -> Optional[float]:
return None return None
def _format_utc_iso(value: datetime) -> str:
return value.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def _parse_datetime(value: Any) -> Optional[datetime]:
raw = str(value or "").strip()
if not raw:
return None
try:
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
except (TypeError, ValueError):
return None
def _normalize_observation_time_contract(city: str, source: str, obs_time: Optional[str]) -> Dict[str, Any]:
parsed = _parse_datetime(obs_time)
if parsed is None:
contract: Dict[str, Any] = {}
tz_name = get_city_timezone_name(city)
if tz_name:
contract["city_timezone"] = tz_name
cadence = SOURCE_CADENCE_SECONDS.get(source)
if cadence is not None:
contract["source_cadence_sec"] = cadence
return contract
if parsed.tzinfo is None:
offset = get_city_utc_offset_seconds(city, parsed.replace(tzinfo=timezone.utc))
local_dt = parsed.replace(tzinfo=timezone(timedelta(seconds=offset)))
observed_utc = (parsed - timedelta(seconds=offset)).replace(tzinfo=timezone.utc)
else:
observed_utc = parsed.astimezone(timezone.utc)
local_dt = city_local_datetime(city, observed_utc)
offset = int(local_dt.utcoffset().total_seconds()) if local_dt.utcoffset() else 0
contract = {
"observed_at_utc": _format_utc_iso(observed_utc),
"observed_at_local": local_dt.replace(microsecond=0).isoformat(),
"city_local_date": local_dt.strftime("%Y-%m-%d"),
"city_utc_offset_seconds": offset,
}
tz_name = get_city_timezone_name(city)
if tz_name:
contract["city_timezone"] = tz_name
cadence = SOURCE_CADENCE_SECONDS.get(source)
if cadence is not None:
contract["source_cadence_sec"] = cadence
return contract
def _iter_runway_points(raw_points: Any) -> Iterable[Dict[str, Any]]: def _iter_runway_points(raw_points: Any) -> Iterable[Dict[str, Any]]:
if isinstance(raw_points, list): if isinstance(raw_points, list):
for item in raw_points: for item in raw_points:
@@ -188,6 +262,25 @@ def normalize_observation_patch(patch: Dict[str, Any]) -> Dict[str, Any]:
if not _has_observation(payload): if not _has_observation(payload):
raise PatchValidationError("patch must include temperature, max, runway, or hourly data") raise PatchValidationError("patch must include temperature, max, runway, or hourly data")
time_contract = _normalize_observation_time_contract(city, source, obs_time)
if time_contract:
payload = {
**payload,
**{
key: value
for key, value in time_contract.items()
if key in {
"observed_at_utc",
"observed_at_local",
"city_local_date",
"city_utc_offset_seconds",
"city_timezone",
"source_cadence_sec",
}
},
}
obs_time = str(time_contract.get("observed_at_utc") or obs_time or "").strip() or None
return { return {
"type": EVENT_TYPE, "type": EVENT_TYPE,
"schema_type": SCHEMA_TYPE, "schema_type": SCHEMA_TYPE,
@@ -195,6 +288,7 @@ def normalize_observation_patch(patch: Dict[str, Any]) -> Dict[str, Any]:
"city": city, "city": city,
"source": source, "source": source,
"obs_time": obs_time, "obs_time": obs_time,
**time_contract,
"ts": int(time.time() * 1000), "ts": int(time.time() * 1000),
"payload": payload, "payload": payload,
} }
+5
View File
@@ -114,6 +114,8 @@ def _build_terminal_row(
"distribution_bias": scan.get("distribution_bias"), "distribution_bias": scan.get("distribution_bias"),
"distribution_preview": scan.get("distribution_preview") or row.get("distribution_preview") or [], "distribution_preview": scan.get("distribution_preview") or row.get("distribution_preview") or [],
"distribution_full": scan.get("distribution_full") or scan.get("distribution_preview") or row.get("distribution_preview") or [], "distribution_full": scan.get("distribution_full") or scan.get("distribution_preview") or row.get("distribution_preview") or [],
"probability_engine": scan.get("probability_engine") or (data.get("probabilities") or {}).get("engine"),
"probability_calibration_mode": scan.get("probability_calibration_mode") or (data.get("probabilities") or {}).get("calibration_mode"),
"model_cluster_sources": daily_entry.get("models") if isinstance(daily_entry.get("models"), dict) else data.get("multi_model", {}).get("forecasts"), "model_cluster_sources": daily_entry.get("models") if isinstance(daily_entry.get("models"), dict) else data.get("multi_model", {}).get("forecasts"),
"window_phase": row.get("window_phase") or scan.get("window_phase"), "window_phase": row.get("window_phase") or scan.get("window_phase"),
"window_score": row.get("window_score") if row.get("window_score") is not None else scan.get("window_score"), "window_score": row.get("window_score") if row.get("window_score") is not None else scan.get("window_score"),
@@ -215,6 +217,9 @@ def _build_quick_row(
} }
), ),
"distribution_preview": distribution[:6] if distribution else [], "distribution_preview": distribution[:6] if distribution else [],
"distribution_full": probs.get("distribution_all") or distribution,
"probability_engine": probs.get("engine"),
"probability_calibration_mode": probs.get("calibration_mode"),
"trading_region": market_region["key"], "trading_region": market_region["key"],
"trading_region_label": market_region["label_en"], "trading_region_label": market_region["label_en"],
"trading_region_label_zh": market_region["label_zh"], "trading_region_label_zh": market_region["label_zh"],