Improve scan terminal freshness and AMSC overlay consistency

frontend/components/dashboard/scan-terminal/__tests__/proxyCachePolicy.test.ts

@@ -82,6 +82,11 @@ export function runTests() {
     /fetchCache:\s*"no-store"/,
     "scan terminal proxy must not put failed or initializing payloads into the Next data cache",
   );
+  assert.match(
+    scanTerminalProxySource,
+    /"since_snapshot_id"/,
+    "scan terminal proxy should forward incremental scan diff snapshot ids to the backend",
+  );
 
   const scanTerminalClientSource = fs.readFileSync(
     path.join(
@@ -108,6 +113,11 @@ export function runTests() {
     /params\.set\("_v",\s*SCAN_TERMINAL_PAYLOAD_VERSION\)/,
     "scan terminal client should vary the CDN cache key without changing backend scan filters",
   );
+  assert.match(
+    scanTerminalClientSource,
+    /params\.set\("diff",\s*"true"\)/,
+    "scan terminal client should request incremental scan diffs when it has a previous snapshot",
+  );
 
   const overviewProxySource = fs.readFileSync(
     path.join(

frontend/components/dashboard/scan-terminal/__tests__/refreshCadencePolicy.test.ts

@@ -36,7 +36,7 @@ async function flushMicrotasks() {
 
 export async function runTests() {
   assert(DASHBOARD_REFRESH_POLICY_MS.observation === 60_000, "observation layer should refresh every 60 seconds");
-  assert(DASHBOARD_REFRESH_POLICY_MS.scanRows === 5 * 60_000, "region/city rows should refresh every 5 minutes");
+  assert(DASHBOARD_REFRESH_POLICY_MS.scanRows === 2 * 60_000, "region/city rows should refresh every 2 minutes");
   assert(DASHBOARD_REFRESH_POLICY_MS.marketOverview === 10 * 60_000, "market overview should refresh every 10 minutes");
   assert(DASHBOARD_REFRESH_POLICY_MS.model === 30 * 60_000, "DEB and multi-model data should refresh every 30 minutes");
   assert(DASHBOARD_REFRESH_POLICY_SEC.metar === 5 * 60, "METAR polling should be 5 minutes");
@@ -69,8 +69,9 @@ export async function runTests() {
     querySource.includes("handleForegroundScanRefresh") &&
       querySource.includes('document.visibilityState !== "visible"') &&
       querySource.includes("SCAN_CACHE_TTL_MS") &&
+      querySource.includes("FOREGROUND_SCAN_REFRESH_AFTER_SUCCESS_MS = 60_000") &&
       querySource.includes("fetchScanTerminal({ forceRefresh: false, showLoading: false })"),
-    "scan list should silently revalidate stale rows when a long-hidden browser tab returns to the foreground",
+    "scan list should silently revalidate rows when a browser tab returns to the foreground after 60 seconds",
   );
   assert(
     chartLogicSource.includes("DASHBOARD_REFRESH_POLICY_MS.metar") &&

frontend/components/dashboard/scan-terminal/__tests__/scanTerminalPatchRows.test.ts

@@ -0,0 +1,156 @@
+import {
+  __applyPatchToScanRowForTest,
+  __mergeScanTerminalIncrementalResponseForTest,
+} from "@/components/dashboard/scan-terminal/use-scan-terminal-query";
+import type { ScanTerminalResponse } from "@/lib/dashboard-types";
+import type { CityPatch } from "@/hooks/use-sse-patches";
+
+function assert(condition: unknown, message: string) {
+  if (!condition) throw new Error(message);
+}
+
+export function runTests() {
+  const patch: CityPatch = {
+    type: "city_observation_patch.v1",
+    city: "shenzhen",
+    revision: 42,
+    changes: {
+      temp: 30.2,
+      max_so_far: 30.4,
+      signed_gap: 1.4,
+      gap_to_target: -1.4,
+      touch_distance: 0,
+      current_reference: 30.4,
+      edge: 0.06,
+      edge_percent: 6,
+      obs_time: "2026-06-14T06:30:00Z",
+    },
+  };
+
+  const next = __applyPatchToScanRowForTest(
+    {
+      city: "shenzhen",
+      current_temp: 29.8,
+      current_max_so_far: 30.0,
+      signed_gap: 1.0,
+      gap_to_target: -1.0,
+      touch_distance: 0.2,
+      current_reference: 30.0,
+      edge: 0.04,
+      edge_percent: 4,
+      local_time: "old",
+      target_threshold: 29,
+    },
+    patch,
+  ) as any;
+
+  assert(next.current_temp === 30.2, "SSE patch should update scan row current temperature");
+  assert(next.current_max_so_far === 30.4, "SSE patch should use backend max_so_far instead of only Math.max(row, temp)");
+  assert(next.signed_gap === 1.4, "SSE patch should update signed_gap when backend sends it");
+  assert(next.gap_to_target === -1.4, "SSE patch should update gap_to_target when backend sends it");
+  assert(next.touch_distance === 0, "SSE patch should update touch_distance when backend sends it");
+  assert(next.current_reference === 30.4, "SSE patch should update current_reference when backend sends it");
+  assert(next.edge === 0.06 && next.edge_percent === 6, "SSE patch should update edge fields when backend sends them");
+  assert(next.local_time === "2026-06-14T06:30:00Z", "SSE patch should update observation time");
+  assert(next.sse_revision === 42, "SSE patch should record the applied revision");
+
+  const derived = __applyPatchToScanRowForTest(
+    {
+      city: "shenzhen",
+      current_temp: 28,
+      current_max_so_far: 28,
+      target_threshold: 30,
+      target_label: "30+",
+    },
+    {
+      type: "city_observation_patch.v1",
+      city: "shenzhen",
+      revision: 43,
+      changes: {
+        temp: 30.5,
+      },
+    },
+  ) as any;
+
+  assert(derived.current_max_so_far === 30.5, "SSE patch should still derive max_so_far from temp when backend omits it");
+  assert(derived.signed_gap === 0.5, "SSE patch should derive signed_gap from updated max and row threshold");
+  assert(derived.gap_to_target === -0.5, "SSE patch should derive gap_to_target from updated max and row threshold");
+  assert(derived.touch_distance === 0, "SSE patch should derive zero touch_distance once threshold is reached");
+
+  const previous: ScanTerminalResponse = {
+    generated_at: "2026-06-14T00:00:00Z",
+    snapshot_id: "scan-old",
+    status: "ready",
+    stale: false,
+    filters: {
+      scan_mode: "tradable",
+      min_price: 0.05,
+      max_price: 0.95,
+      min_edge_pct: 2,
+      min_liquidity: 500,
+      high_liquidity_only: false,
+      market_type: "maxtemp",
+      time_range: "today",
+      limit: 180,
+    },
+    summary: {
+      recommended_count: 2,
+      visible_count: 2,
+      candidate_total: 2,
+      tradable_market_count: 2,
+      total_volume: 1000,
+    },
+    top_signal: null,
+    rows: [
+      { id: "row-1", rank: 1, city: "shenzhen", edge_percent: 4 },
+      { id: "row-removed", rank: 2, city: "paris", edge_percent: 3 },
+    ],
+  };
+
+  const notModified = __mergeScanTerminalIncrementalResponseForTest(previous, {
+    generated_at: "2026-06-14T00:01:00Z",
+    snapshot_id: "scan-old",
+    status: "not_modified",
+    stale: false,
+    filters: previous.filters,
+    summary: previous.summary,
+    top_signal: previous.top_signal,
+    rows: [],
+    diff: {
+      mode: "not_modified",
+      base_snapshot_id: "scan-old",
+      snapshot_id: "scan-old",
+      rows_changed: [],
+      removed_row_ids: [],
+    },
+  });
+
+  assert(notModified.rows.length === 2, "not_modified scan diff should keep the previous rows");
+  assert(notModified.status === "ready", "not_modified scan diff should normalize to a renderable ready payload");
+
+  const delta = __mergeScanTerminalIncrementalResponseForTest(previous, {
+    generated_at: "2026-06-14T00:02:00Z",
+    snapshot_id: "scan-new",
+    status: "ready",
+    stale: false,
+    filters: previous.filters,
+    summary: { ...previous.summary, recommended_count: 2 },
+    top_signal: { id: "row-added", city: "tokyo" } as any,
+    rows: [],
+    diff: {
+      mode: "row_delta",
+      base_snapshot_id: "scan-old",
+      snapshot_id: "scan-new",
+      rows_changed: [
+        { id: "row-1", rank: 1, city: "shenzhen", edge_percent: 6 },
+        { id: "row-added", rank: 2, city: "tokyo", edge_percent: 5 },
+      ],
+      removed_row_ids: ["row-removed"],
+    },
+  });
+
+  assert(delta.snapshot_id === "scan-new", "row_delta scan diff should advance snapshot_id");
+  assert(delta.rows.length === 2, "row_delta scan diff should merge changed, added, and removed rows");
+  assert(delta.rows[0].id === "row-1" && delta.rows[0].edge_percent === 6, "row_delta scan diff should replace changed rows");
+  assert(delta.rows[1].id === "row-added", "row_delta scan diff should append new rows in rank order");
+}

frontend/components/dashboard/scan-terminal/scan-terminal-client.ts

@@ -27,6 +27,7 @@ const SCAN_TERMINAL_PAYLOAD_VERSION = "runway-slim-v1";
 type TerminalQueryOptions = {
   forceRefresh?: boolean;
   signal?: AbortSignal;
+  sinceSnapshotId?: string | null;
   timezoneOffsetSeconds?: number | null;
   tradingRegion?: string;
 };
@@ -123,6 +124,7 @@ async function readJsonOrThrow<T>(path: string, init?: RequestInit): Promise<T>
 async function getTerminal({
   forceRefresh = false,
   signal,
+  sinceSnapshotId,
   timezoneOffsetSeconds,
   tradingRegion,
 }: TerminalQueryOptions = {}) {
@@ -143,6 +145,11 @@ async function getTerminal({
   if (Number.isFinite(timezoneOffsetSeconds)) {
     params.set("timezone_offset_seconds", String(Math.trunc(Number(timezoneOffsetSeconds))));
   }
+  const trimmedSnapshotId = String(sinceSnapshotId || "").trim();
+  if (!forceRefresh && trimmedSnapshotId) {
+    params.set("diff", "true");
+    params.set("since_snapshot_id", trimmedSnapshotId);
+  }
   params.set("_v", SCAN_TERMINAL_PAYLOAD_VERSION);
   if (forceRefresh) {
     params.set("_ts", String(Date.now()));

frontend/components/dashboard/scan-terminal/use-scan-terminal-query.ts

@@ -13,11 +13,24 @@ import {
   useSsePatchVersion,
   type CityPatch,
 } from "@/hooks/use-sse-patches";
-import type { ScanTerminalResponse } from "@/lib/dashboard-types";
+import type {
+  ScanOpportunityRow,
+  ScanTerminalResponse,
+} from "@/lib/dashboard-types";
 
 const SCAN_CACHE_PREFIX = "polyweather_scan_v2";
 const SCAN_CACHE_TTL_MS = DASHBOARD_REFRESH_POLICY_MS.scanRows;
+const FOREGROUND_SCAN_REFRESH_AFTER_SUCCESS_MS = 60_000;
 const MAX_STALE_SCAN_CACHE_MS = 6 * 60 * 60 * 1000;
+const DERIVED_SCAN_PATCH_NUMBER_FIELDS = [
+  "signed_gap",
+  "gap_to_target",
+  "touch_distance",
+  "current_reference",
+  "edge",
+  "edge_percent",
+  "deb_prediction",
+] as const;
 
 function scanCacheKey(tradingRegion: string): string {
   return `${SCAN_CACHE_PREFIX}:${tradingRegion || "all"}`;
@@ -48,23 +61,82 @@ function normalizeCityKey(city: string | null | undefined) {
   return String(city || "").trim().toLowerCase();
 }
 
-function applyPatchToScanRow(row: any, patch: CityPatch) {
-  const temp = typeof patch.changes.temp === "number" && Number.isFinite(patch.changes.temp)
-    ? patch.changes.temp
-    : null;
-  if (temp === null) return row;
+function finiteNumber(value: unknown): number | null {
+  const number = Number(value);
+  return Number.isFinite(number) ? number : null;
+}
+
+function firstFiniteNumber(...values: unknown[]) {
+  for (const value of values) {
+    const number = finiteNumber(value);
+    if (number !== null) return number;
+  }
+  return null;
+}
+
+function isBelowTargetRow(row: any) {
+  const text = String(
+    row?.market_direction ||
+      row?.temperature_direction ||
+      row?.market_question ||
+      row?.target_label ||
+      row?.action ||
+      "",
+  ).toLowerCase();
+  return /\b(below|under|lte|less)\b|<=|≤|以下/.test(text);
+}
+
+function deriveTargetGapFields(row: any, maxSoFar: number | null) {
+  const target = firstFiniteNumber(
+    row?.target_threshold,
+    row?.target_value,
+    row?.target_lower,
+    row?.target_upper,
+  );
+  if (target === null || maxSoFar === null) return null;
+
+  const belowTargetRow = isBelowTargetRow(row);
+  const signedGap = belowTargetRow ? target - maxSoFar : maxSoFar - target;
+  const touchDistance = belowTargetRow
+    ? Math.max(0, maxSoFar - target)
+    : Math.max(0, target - maxSoFar);
   return {
+    signed_gap: Number(signedGap.toFixed(2)),
+    gap_to_target: Number((-signedGap).toFixed(2)),
+    touch_distance: Number(touchDistance.toFixed(2)),
+  };
+}
+
+function applyPatchToScanRow(row: any, patch: CityPatch) {
+  const temp = finiteNumber(patch.changes.temp);
+  const patchMaxSoFar = finiteNumber(patch.changes.max_so_far);
+  if (temp === null) return row;
+  const currentMaxSoFar = firstFiniteNumber(row.current_max_so_far, temp) ?? temp;
+  const maxSoFar = patchMaxSoFar ?? Math.max(temp, currentMaxSoFar);
+  const derivedGapFields = deriveTargetGapFields(row, maxSoFar);
+  const nextRow: any = {
     ...row,
     current_temp: temp,
-    current_max_so_far: Math.max(
-      temp,
-      typeof row.current_max_so_far === "number" && Number.isFinite(row.current_max_so_far)
-        ? row.current_max_so_far
-        : temp,
-    ),
+    current_max_so_far: maxSoFar,
     local_time: typeof patch.changes.obs_time === "string" ? patch.changes.obs_time : row.local_time,
     sse_revision: patch.revision,
   };
+  for (const key of DERIVED_SCAN_PATCH_NUMBER_FIELDS) {
+    const value = finiteNumber(patch.changes[key]);
+    if (value !== null) {
+      nextRow[key] = value;
+    }
+  }
+  if (derivedGapFields) {
+    for (const key of ["signed_gap", "gap_to_target", "touch_distance"] as const) {
+      if (finiteNumber(patch.changes[key]) === null) {
+        nextRow[key] = derivedGapFields[key];
+      }
+    }
+  }
+  return {
+    ...nextRow,
+  };
 }
 
 function applyTerminalPatches(
@@ -82,6 +154,76 @@ function applyTerminalPatches(
   return changed ? { ...data, rows } : data;
 }
 
+function scanRowId(row: ScanOpportunityRow | null | undefined) {
+  const id = row?.id;
+  return typeof id === "string" && id ? id : null;
+}
+
+function sortScanRowsByRank(rows: ScanOpportunityRow[]) {
+  return [...rows].sort((a, b) => {
+    const rankA = finiteNumber(a.rank);
+    const rankB = finiteNumber(b.rank);
+    if (rankA !== null && rankB !== null && rankA !== rankB) {
+      return rankA - rankB;
+    }
+    if (rankA !== null && rankB === null) return -1;
+    if (rankA === null && rankB !== null) return 1;
+    return 0;
+  });
+}
+
+function mergeScanTerminalIncrementalResponse(
+  previous: ScanTerminalResponse | null | undefined,
+  next: ScanTerminalResponse,
+): ScanTerminalResponse {
+  const diff = next.diff;
+  if (!diff || diff.mode === "full") return next;
+  if (!previous?.snapshot_id) return next;
+
+  const baseSnapshotId = diff.base_snapshot_id || next.snapshot_id;
+  if (previous.snapshot_id !== baseSnapshotId) return next;
+
+  if (diff.mode === "not_modified") {
+    return {
+      ...previous,
+      ...next,
+      status: "ready",
+      stale: false,
+      rows: previous.rows,
+    };
+  }
+
+  if (diff.mode !== "row_delta") return next;
+
+  const removedIds = new Set((diff.removed_row_ids || []).map((id) => String(id)));
+  const changedRows = new Map<string, ScanOpportunityRow>();
+  for (const row of diff.rows_changed || []) {
+    const id = scanRowId(row);
+    if (id) changedRows.set(id, row);
+  }
+
+  const seenIds = new Set<string>();
+  const mergedRows: ScanOpportunityRow[] = [];
+  for (const row of previous.rows || []) {
+    const id = scanRowId(row);
+    if (!id || removedIds.has(id)) continue;
+    const replacement = changedRows.get(id);
+    mergedRows.push(replacement || row);
+    seenIds.add(id);
+  }
+  for (const [id, row] of changedRows.entries()) {
+    if (!seenIds.has(id) && !removedIds.has(id)) {
+      mergedRows.push(row);
+    }
+  }
+
+  return {
+    ...previous,
+    ...next,
+    rows: sortScanRowsByRank(mergedRows),
+  };
+}
+
 export function useScanTerminalQuery({
   isPro,
   proAccessLoading,
@@ -114,6 +256,11 @@ export function useScanTerminalQuery({
     }
     return null;
   });
+  const latestScanDataRef = useRef<ScanTerminalResponse | null>(null);
+
+  useEffect(() => {
+    latestScanDataRef.current = terminalData || cachedRows;
+  }, [terminalData, cachedRows]);
 
   useEffect(() => {
     if (typeof window === "undefined") return;
@@ -136,13 +283,17 @@ export function useScanTerminalQuery({
         lastForcedScanRefreshAtRef.current = Date.now();
       }
       await run({
-        request: (signal) =>
-          scanTerminalClient.getTerminal({
+        request: async (signal) => {
+          const previous = latestScanDataRef.current;
+          const next = await scanTerminalClient.getTerminal({
             forceRefresh,
             signal,
+            sinceSnapshotId: !forceRefresh ? previous?.snapshot_id : null,
             timezoneOffsetSeconds,
             tradingRegion,
-          }),
+          });
+          return mergeScanTerminalIncrementalResponse(previous, next);
+        },
         showLoading,
         onSuccess: (data) => {
           lastScanSuccessAtRef.current = Date.now();
@@ -191,7 +342,7 @@ export function useScanTerminalQuery({
       if (now - lastForegroundScanRefreshAtRef.current < 30_000) return;
       if (
         lastScanSuccessAtRef.current > 0 &&
-        now - lastScanSuccessAtRef.current < SCAN_CACHE_TTL_MS
+        now - lastScanSuccessAtRef.current < FOREGROUND_SCAN_REFRESH_AFTER_SUCCESS_MS
       ) {
         return;
       }
@@ -268,3 +419,7 @@ export function useScanTerminalQuery({
     terminalData: effectiveData,
   };
 }
+
+export const __applyPatchToScanRowForTest = applyPatchToScanRow;
+export const __mergeScanTerminalIncrementalResponseForTest =
+  mergeScanTerminalIncrementalResponse;