feat: implement real-time SSE event architecture with Redis stream integration and add associated validation tests

2026-05-27 11:03:04 +08:00
parent 820dabfbf3
commit 573768846e
18 changed files with 1379 additions and 22 deletions
@@ -30,6 +30,7 @@ import {
  normObs,
  prefersHighFrequencyRunwayResolution,
  readSessionCache,
+  selectDisplayRunwayTemp,
  seedHourlyForecastFromRow,
  shouldPollLiveChart,
  validNumber,
@@ -478,7 +479,7 @@ export function LiveTemperatureThresholdChart({
    () => getObservationDisplayMetrics(row, chartHourly, settlementPlate),
    [row, chartHourly, settlementPlate],
  );
-  const displayRunwayTemp = liveTemp ?? currentRunwayTemp;
+  const displayRunwayTemp = selectDisplayRunwayTemp(liveTemp, currentRunwayTemp, hasRunwayData);
  const wundergroundDailyHigh = validNumber(chartHourly?.airportCurrent?.max_so_far ?? chartHourly?.airportPrimary?.max_so_far) ?? null;

  const localDateStr = chartLocalDate || new Date().toISOString().slice(0, 10);
@@ -786,3 +787,4 @@ export const __getObservationDisplayMetricsForTest = getObservationDisplayMetric
 export const __getPeakGlowStateForTest = getPeakGlowState;
 export const __shouldPollLiveChartForTest = shouldPollLiveChart;
 export const __mergePatchIntoHourlyForTest = mergePatchIntoHourly;
+export const __selectDisplayRunwayTempForTest = selectDisplayRunwayTemp;
@@ -42,6 +42,20 @@ export function runTests() {
  assert(store.includes("replay_events"), "event store must expose replay_events");
  assert(store.includes("replay_requires_resync"), "event store must detect incomplete replay windows");

+  const redisStorePath = path.join(repoRoot, "web", "redis_realtime_event_store.py");
+  assert(fs.existsSync(redisStorePath), "backend must define a Redis Stream realtime event store");
+  const redisStore = fs.readFileSync(redisStorePath, "utf8");
+  assert(redisStore.includes("RedisRealtimeEventStore"), "Redis store must expose RedisRealtimeEventStore");
+  assert(redisStore.includes("XADD") && redisStore.includes("MAXLEN"), "Redis store must append patches to a bounded Redis Stream");
+  assert(redisStore.includes("xread"), "Redis store must support live fanout through Redis Stream reads");
+  assert(redisStore.includes("counter:city_observation_revision"), "Redis store must keep a numeric revision counter for frontend compatibility");
+
+  const storeFactoryPath = path.join(repoRoot, "web", "realtime_event_store_factory.py");
+  assert(fs.existsSync(storeFactoryPath), "backend must define a realtime event store factory");
+  const storeFactory = fs.readFileSync(storeFactoryPath, "utf8");
+  assert(storeFactory.includes("POLYWEATHER_EVENT_STORE"), "event store factory must select sqlite/redis from runtime config");
+  assert(storeFactory.includes("POLYWEATHER_REDIS_REQUIRED"), "event store factory must support strict Redis mode");
+
  const sseRouterPath = path.join(repoRoot, "web", "routers", "sse_router.py");
  assert(fs.existsSync(sseRouterPath), "FastAPI backend must define web/routers/sse_router.py");
  const sseRouter = fs.readFileSync(sseRouterPath, "utf8");
@@ -53,6 +67,9 @@ export function runTests() {
  assert(sseRouter.includes('"/api/internal/collector-patch"'), "SSE router must expose collector patch ingest endpoint");
  assert(sseRouter.includes("StreamingResponse"), "SSE route must return StreamingResponse");
  assert(sseRouter.includes('"text/event-stream"'), "SSE route must use text/event-stream media type");
+  assert(sseRouter.includes("create_realtime_event_store"), "SSE router must use the realtime event store factory");
+  assert(sseRouter.includes("_ensure_live_subscription"), "SSE router must start external live fanout when the store provides it");
+  assert(sseRouter.includes("uses_external_live_fanout"), "Redis-backed ingest must not directly broadcast duplicate local events");

  const appFactory = readRepoFile("web", "app_factory.py");
  assert(appFactory.includes("sse_router"), "FastAPI app factory must register the SSE router");
@@ -8,6 +8,7 @@ import {
  __getVisibleTemperatureSeriesForTest,
  __isTemperatureSeriesVisibleByDefaultForTest,
  __mergePatchIntoHourlyForTest,
+  __selectDisplayRunwayTempForTest,
 } from "@/components/dashboard/scan-terminal/LiveTemperatureThresholdChart";

 function assert(condition: unknown, message: string): asserts condition {
@@ -396,6 +397,46 @@ export function runTests() {
    "Shenzhen HKO observation series should include the airportPrimaryTodayObs curve points",
  );

+  const hongKongCowinAndHko = __buildTemperatureChartDataForTest(
+    {
+      city: "hong kong",
+      local_date: "2026-05-27",
+      local_time: "10:42",
+      tz_offset_seconds: 8 * 60 * 60,
+      temp_symbol: "°C",
+    } as any,
+    {
+      localTime: "10:42",
+      times: ["00:00", "12:00", "18:00"],
+      temps: [27.2, 30.9, 27.6],
+      airportPrimary: {
+        source_code: "cowin_obs",
+        source_label: "CoWIN 6087",
+        station_label: "保良局陳守仁小學 1min (CoWIN)",
+        temp: 31.3,
+        obs_time: "2026-05-27T02:42:00Z",
+      },
+      airportPrimaryTodayObs: [
+        ["2026-05-27T02:40:00Z", 31.1],
+        ["2026-05-27T02:41:00Z", 31.2],
+        ["2026-05-27T02:42:00Z", 31.3],
+      ],
+      settlementTodayObs: [
+        { time: "2026-05-27T02:30:00Z", temp: 31.0 },
+        { time: "2026-05-27T02:40:00Z", temp: 31.2 },
+      ],
+    } as any,
+    "1D",
+  );
+  const hongKongCowinSeries = seriesByKey(hongKongCowinAndHko.series, "settlement") as any;
+  const hongKongHkoSeries = seriesByKey(hongKongCowinAndHko.series, "madis") as any;
+  assert(hongKongCowinSeries?.label === "CoWIN 6087", "Hong Kong should render CoWIN 6087 as the reference-station curve");
+  assert(
+    hongKongCowinSeries.values.filter((value: number | null) => value !== null).length >= 2,
+    "Hong Kong CoWIN 6087 curve should use airportPrimaryTodayObs history points",
+  );
+  assert(hongKongHkoSeries?.label === "HKO", "Hong Kong HKO settlement observations should remain visible as the HKO curve");
+
  const chengduFromAmosSnapshot = __buildTemperatureChartDataForTest(
    {
      city: "chengdu",
@@ -663,6 +704,49 @@ export function runTests() {
    "AMOS temp/dew tuples should not be misread as two runway temperature samples",
  );

+  const seoulRunwayMetrics = __getObservationDisplayMetricsForTest(
+    {
+      city: "seoul",
+      local_date: "2026-05-27",
+      local_time: "11:45",
+      tz_offset_seconds: 9 * 60 * 60,
+      current_temp: 23.0,
+      temp_symbol: "°C",
+    } as any,
+    {
+      localTime: "11:45",
+      times: ["00:00", "12:00", "18:00", "23:00"],
+      temps: [22.6, 22.6, 22.0, 21.4],
+      runwayPlateHistory: {
+        "15R/33L": [
+          { time: "2026-05-27T02:40:00Z", temp: 23.9 },
+          { time: "2026-05-27T02:45:00Z", temp: 24.3 },
+        ],
+        "16L/34R": [
+          { time: "2026-05-27T02:40:00Z", temp: 24.1 },
+          { time: "2026-05-27T02:45:00Z", temp: 24.7 },
+        ],
+      },
+      amos: {
+        source: "amos",
+        temp_c: 23.0,
+      },
+    } as any,
+    { maxTemp: 23.0 },
+  );
+  assert(
+    seoulRunwayMetrics.currentRunwayTemp === 24.3,
+    "runway header should use the latest settlement runway point instead of AMOS/METAR aggregate temp",
+  );
+  assert(
+    seoulRunwayMetrics.observedHighRunway === 24.3,
+    "runway high should follow settlement runway history before AMOS/METAR aggregate temp",
+  );
+  assert(
+    __selectDisplayRunwayTempForTest(23.0, 24.3, true) === 24.3,
+    "live aggregate temp should not override runway-history current temp when runway data is rendered",
+  );
+
  const newYorkMetrics = __getObservationDisplayMetricsForTest(
    {
      city: "new york",
@@ -548,6 +548,35 @@ function maxObservationValue(obs: Array<{ ts: number; value: number }>) {
  return Math.max(...obs.map((point) => point.value));
 }

+function getRunwayHistoryObservationMetrics(
+  row: ScanOpportunityRow | null,
+  hourly: HourlyForecast,
+) {
+  const tzOffset = row?.tz_offset_seconds ?? 0;
+  const localDateStr = resolveChartLocalDate(row, hourly);
+  const localDayBounds = getLocalDayBounds(localDateStr);
+  const runwayHistorySeries = buildRunwayHistorySeries(row, hourly, tzOffset, localDateStr, 1)
+    .map((item) => ({
+      ...item,
+      points: filterTimelinePointsToLocalDay(item.points, localDayBounds),
+    }))
+    .filter((item) => item.points.length > 0);
+
+  const settlementSeries = runwayHistorySeries.filter((item) => item.isSettlement);
+  const candidateSeries = settlementSeries.length ? settlementSeries : runwayHistorySeries;
+  const points = candidateSeries.flatMap((item) => item.points);
+  if (!points.length) return { latest: null, high: null };
+
+  const latestTs = Math.max(...points.map((point) => point.ts));
+  const latestValues = points
+    .filter((point) => point.ts === latestTs)
+    .map((point) => point.value);
+  return {
+    latest: latestValues.length ? Math.max(...latestValues) : null,
+    high: Math.max(...points.map((point) => point.value)),
+  };
+}
+
 function hasRenderableLineSeries(series: EvidenceSeries[]) {
  return series.some(
    (item) => item.values.filter((value) => validNumber(value) !== null).length >= 2,
@@ -588,6 +617,7 @@ function getObservationDisplayMetrics(
  const airportCurrentTemp = validNumber(hourly?.airportCurrent?.temp) ?? validNumber(hourly?.airportPrimary?.temp);
  const airportHigh = validNumber(hourly?.airportCurrent?.max_so_far) ?? validNumber(hourly?.airportPrimary?.max_so_far);
  const rowMetarHigh = validNumber(row?.metar_context?.airport_max_so_far ?? row?.metar_context?.max_temp ?? row?.current_max_so_far);
+  const runwayHistoryMetrics = getRunwayHistoryObservationMetrics(row, hourly);

  const settlementCityKey = normalizeCityKey(row?.city);
  const isShenzhen = settlementCityKey === 'shenzhen';
@@ -616,14 +646,16 @@ function getObservationDisplayMetrics(
      null;
  } else {
    currentRunwayTemp =
-      validNumber(hourly?.amos?.temp_c) ??
+      runwayHistoryMetrics.latest ??
      settlementPlate?.maxTemp ??
+      validNumber(hourly?.amos?.temp_c) ??
      latestSettlement ??
      latestMetar ??
      airportCurrentTemp ??
      validNumber(row?.current_temp) ??
      null;
    observedHighRunway =
+      runwayHistoryMetrics.high ??
      settlementPlate?.maxTemp ??
      highSettlement ??
      airportHigh ??
@@ -638,6 +670,17 @@ function getObservationDisplayMetrics(
  return { currentRunwayTemp, observedHighMetar, observedHighRunway };
 }

+function selectDisplayRunwayTemp(
+  liveTemp: number | null,
+  currentRunwayTemp: number | null,
+  hasRunwayData: boolean,
+) {
+  if (hasRunwayData && currentRunwayTemp !== null) {
+    return currentRunwayTemp;
+  }
+  return liveTemp ?? currentRunwayTemp;
+}
+
 function isSettlementRunway(row: ScanOpportunityRow | null, rwy: string) {
  const cityKey = normalizeCityKey(row?.city);
  const settlementPairs = SETTLEMENT_RUNWAY_PAIRS[cityKey] || [];
@@ -1071,6 +1114,7 @@ function buildRunwayHistorySeries(
  hourly: HourlyForecast,
  tzOffset: number,
  localDateStr: string,
+  minPoints = 2,
 ): RunwayHistorySeries[] {
  const directHistory =
    hourly?.runwayPlateHistory ??
@@ -1100,7 +1144,7 @@ function buildRunwayHistorySeries(
          points,
        };
      })
-      .filter((series) => series.points.length > 1);
+      .filter((series) => series.points.length >= minPoints);
    if (directSeries.length) return directSeries;
  }

@@ -1153,7 +1197,7 @@ function buildRunwayHistorySeries(
          };
        })
        .filter((point) => validNumber(point.value) !== null);
-      if (values.length <= 1) return null;
+      if (values.length < minPoints) return null;
      return {
        key: runwaySeriesKey(rwy),
        label: `${rwy}${isSettlement ? " 结算跑道" : ""}`,
@@ -2047,6 +2091,7 @@ export {
  normalizeCityKey,
  prefersHighFrequencyRunwayResolution,
  readSessionCache,
+  selectDisplayRunwayTemp,
  seedHourlyForecastFromRow,
  seriesStats,
  shouldPollLiveChart,