Compare commits

...

33 Commits

Author SHA1 Message Date
2569718930@qq.com 49cb0bb42d Prevent terminal auth spinner deadlock 2026-06-01 12:36:29 +08:00
2569718930@qq.com fb6c354317 Avoid terminal full-detail fallback overload 2026-06-01 12:06:46 +08:00
2569718930@qq.com fee402145e Slim terminal chart detail batches 2026-06-01 11:37:16 +08:00
2569718930@qq.com d1633331bc Load terminal chart canvas eagerly 2026-06-01 07:05:08 +08:00
2569718930@qq.com 6e00160897 Version scan terminal cache key 2026-06-01 06:46:34 +08:00
2569718930@qq.com 6bc53c6e5d Slim deferred scan runway history 2026-06-01 06:25:27 +08:00
2569718930@qq.com 7625a99020 Cache terminal city registry fallback 2026-06-01 06:09:44 +08:00
2569718930@qq.com 3447a043ff Preload terminal chart canvas 2026-06-01 05:56:09 +08:00
2569718930@qq.com 10cf557b11 Batch deferred chart detail loads 2026-06-01 02:56:13 +08:00
2569718930@qq.com eaf17c7a32 Align scan prewarm observability test 2026-06-01 02:35:23 +08:00
2569718930@qq.com 14526d6ce8 Prewarm default scan terminal payload 2026-06-01 02:06:40 +08:00
2569718930@qq.com 9a0d7ae444 Persist scan terminal cache in Redis 2026-06-01 01:53:54 +08:00
2569718930@qq.com de5083ef18 Defer secondary chart detail loads 2026-06-01 01:27:53 +08:00
2569718930@qq.com 2424b68532 Compact scan runway history payload 2026-06-01 01:03:33 +08:00
2569718930@qq.com ff2a3da100 Use longer timeout for scan prewarm 2026-06-01 00:30:25 +08:00
2569718930@qq.com 7040598e26 Prewarm terminal scan payload 2026-06-01 00:14:12 +08:00
2569718930@qq.com 22f4ac1f26 Preserve scan cache on timeout 2026-05-31 23:54:35 +08:00
2569718930@qq.com 3e5ff4ea8f Isolate scan prewarm to web service 2026-05-31 23:31:32 +08:00
2569718930@qq.com 39c7c1903c Shorten cold scan terminal timeout 2026-05-31 22:54:43 +08:00
2569718930@qq.com eecaa48ec2 Limit city detail fallback to batch failures 2026-05-31 22:34:28 +08:00
2569718930@qq.com d911af1225 Reduce duplicate terminal detail loads 2026-05-31 22:14:17 +08:00
2569718930@qq.com 38c6cefb27 Direct resync stale SSE clients 2026-05-31 21:48:09 +08:00
2569718930@qq.com a3df2e00dc Clamp SSE replay window on backend 2026-05-31 21:33:35 +08:00
2569718930@qq.com fde69123cd Merge branch 'codex/signup-trial-reconcile' 2026-05-31 21:25:58 +08:00
2569718930@qq.com 2d963c16f3 Coalesce terminal chart detail loads 2026-05-31 21:17:38 +08:00
2569718930@qq.com 071d749ef0 Reduce terminal detail refresh traffic 2026-05-31 21:01:11 +08:00
2569718930@qq.com 8d552a9279 Scale SSE replay window by visible cities 2026-05-31 20:33:04 +08:00
2569718930@qq.com 9c5a08dc1e Reduce false signup trial risk alerts 2026-05-31 20:21:57 +08:00
2569718930@qq.com ada5f274d3 Debounce terminal SSE subscription reconnects 2026-05-31 20:15:39 +08:00
2569718930@qq.com 78ea0326a5 Return partial city detail batches 2026-05-31 19:51:28 +08:00
2569718930@qq.com d3f444dbf6 Bind PolyWeather services to loopback 2026-05-31 19:46:00 +08:00
2569718930@qq.com 8d26afdec0 Reduce terminal auth and analytics stalls 2026-05-31 19:21:53 +08:00
2569718930@qq.com 5083b7c433 Instrument API timing and reduce detail fallbacks 2026-05-31 18:58:57 +08:00
50 changed files with 3566 additions and 388 deletions
+8 -2
View File
@@ -28,6 +28,9 @@ services:
env_file: &id001 env_file: &id001
- .env - .env
environment: environment:
POLYWEATHER_REDIS_URL: ${POLYWEATHER_REDIS_URL:-redis://polyweather_redis:6379/0}
POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED: 'false'
POLYWEATHER_SERVICE_ROLE: bot
TELEGRAM_AIRPORT_PUSH_INTERVAL_SEC: ${POLYWEATHER_BOT_AIRPORT_PUSH_INTERVAL_SEC:-180} TELEGRAM_AIRPORT_PUSH_INTERVAL_SEC: ${POLYWEATHER_BOT_AIRPORT_PUSH_INTERVAL_SEC:-180}
TELEGRAM_AIRPORT_PUSH_MAX_WORKERS: ${POLYWEATHER_BOT_AIRPORT_PUSH_MAX_WORKERS:-1} TELEGRAM_AIRPORT_PUSH_MAX_WORKERS: ${POLYWEATHER_BOT_AIRPORT_PUSH_MAX_WORKERS:-1}
healthcheck: healthcheck:
@@ -79,7 +82,7 @@ services:
timeout: 5s timeout: 5s
image: ghcr.io/yangyuan-zhen/polyweather-frontend:${IMAGE_TAG:-latest} image: ghcr.io/yangyuan-zhen/polyweather-frontend:${IMAGE_TAG:-latest}
ports: ports:
- 3001:3000 - "127.0.0.1:3001:3000"
restart: unless-stopped restart: unless-stopped
polyweather_web: polyweather_web:
command: python web/app.py command: python web/app.py
@@ -93,6 +96,9 @@ services:
max-file: "3" max-file: "3"
container_name: polyweather_web container_name: polyweather_web
env_file: *id001 env_file: *id001
environment:
POLYWEATHER_REDIS_URL: ${POLYWEATHER_REDIS_URL:-redis://polyweather_redis:6379/0}
POLYWEATHER_SERVICE_ROLE: web
healthcheck: healthcheck:
interval: 30s interval: 30s
retries: 3 retries: 3
@@ -104,7 +110,7 @@ services:
timeout: 5s timeout: 5s
image: ghcr.io/yangyuan-zhen/polyweather-backend:${IMAGE_TAG:-latest} image: ghcr.io/yangyuan-zhen/polyweather-backend:${IMAGE_TAG:-latest}
ports: ports:
- 8000:8000 - "127.0.0.1:8000:8000"
restart: unless-stopped restart: unless-stopped
user: ${UID:-1000}:${GID:-1000} user: ${UID:-1000}:${GID:-1000}
volumes: volumes:
+1 -1
View File
@@ -192,7 +192,7 @@ Ops
- 支付相关路由:`no-store` - 支付相关路由:`no-store`
- 当 detail 缓存只返回单模型或单日 forecast 时,前端会自动强刷完整 detail,并在补齐前显示同步提示 / 占位卡 - 当 detail 缓存只返回单模型或单日 forecast 时,前端会自动强刷完整 detail,并在补齐前显示同步提示 / 占位卡
- 今日日内分析打开时如果正在切换城市、日期或 detail 深度,弹窗会阻断旧内容点击并显示刷新锁 - 今日日内分析打开时如果正在切换城市、日期或 detail 深度,弹窗会阻断旧内容点击并显示刷新锁
- 终端图表订阅 `/api/events?cities=...&since_revision=...&replay_limit=500`,接收 `city_observation_patch.v1`;无 patch 超过 2 分钟时,可见图表才触发 60 秒兜底刷新 - 终端图表订阅 `/api/events?cities=...&since_revision=...&replay_limit=按可见城市数动态限制`,接收 `city_observation_patch.v1`;无 patch 超过 2 分钟时,可见图表才触发 60 秒兜底刷新
- 前端只消费 HTTP snapshot + SSE patch,不直接感知 RedisRedis Stream / SQLite event log 都由后端统一封装 - 前端只消费 HTTP snapshot + SSE patch,不直接感知 RedisRedis Stream / SQLite event log 都由后端统一封装
## Vercel 节流建议 ## Vercel 节流建议
+73 -17
View File
@@ -7,25 +7,46 @@ import {
buildProxyExceptionResponse, buildProxyExceptionResponse,
buildUpstreamErrorResponse, buildUpstreamErrorResponse,
} from "@/lib/api-proxy"; } from "@/lib/api-proxy";
import {
createProxyTimer,
finishProxyTimedResponse,
} from "@/lib/proxy-timing";
const API_BASE = process.env.POLYWEATHER_API_BASE_URL; const API_BASE = process.env.POLYWEATHER_API_BASE_URL;
const ANALYTICS_ENABLED = const ANALYTICS_ENABLED =
process.env.NEXT_PUBLIC_POLYWEATHER_APP_ANALYTICS !== "false"; process.env.NEXT_PUBLIC_POLYWEATHER_APP_ANALYTICS !== "false";
const ANALYTICS_PROXY_TIMEOUT_MS = Math.max(
250,
Number(process.env.POLYWEATHER_ANALYTICS_PROXY_TIMEOUT_MS || "1500") || 1500,
);
export async function POST(req: NextRequest) { export async function POST(req: NextRequest) {
const timer = createProxyTimer(req, "analytics_events");
if (!ANALYTICS_ENABLED) { if (!ANALYTICS_ENABLED) {
return new NextResponse(null, { status: 204 }); return finishProxyTimedResponse(
} new NextResponse(null, { status: 204 }),
timer,
if (!API_BASE) { "disabled",
return NextResponse.json(
{ error: "POLYWEATHER_API_BASE_URL is not configured" },
{ status: 500 },
); );
} }
if (!API_BASE) {
return finishProxyTimedResponse(
NextResponse.json(
{ error: "POLYWEATHER_API_BASE_URL is not configured" },
{ status: 500 },
),
timer,
"missing_api_base",
);
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), ANALYTICS_PROXY_TIMEOUT_MS);
let auth: Awaited<ReturnType<typeof buildBackendRequestHeaders>> | null = null;
try { try {
const body = await req.json(); const body = await timer.measure("request_read", () => req.json());
const payload = const payload =
body && typeof body.payload === "object" && body.payload != null body && typeof body.payload === "object" && body.payload != null
? body.payload ? body.payload
@@ -42,30 +63,65 @@ export async function POST(req: NextRequest) {
referer_header: req.headers.get("referer") || "", referer_header: req.headers.get("referer") || "",
}, },
}; };
const auth = await buildBackendRequestHeaders(req, { auth = await timer.measure("auth_headers", () =>
buildBackendRequestHeaders(req, {
includeSupabaseIdentity: false, includeSupabaseIdentity: false,
}); }),
);
const headers = new Headers(auth.headers); const headers = new Headers(auth.headers);
headers.set("Content-Type", "application/json"); headers.set("Content-Type", "application/json");
const res = await fetch(`${API_BASE}/api/analytics/events`, { const res = await timer.measure("backend_fetch", () =>
fetch(`${API_BASE}/api/analytics/events`, {
method: "POST", method: "POST",
headers, headers,
body: JSON.stringify(enrichedBody), body: JSON.stringify(enrichedBody),
cache: "no-store", cache: "no-store",
}); signal: controller.signal,
}),
);
const backendServerTiming = res.headers.get("server-timing") || "";
if (!res.ok) { if (!res.ok) {
const raw = await res.text(); const raw = await timer.measure("backend_read", () => res.text());
const response = buildUpstreamErrorResponse(res.status, raw, { const response = buildUpstreamErrorResponse(res.status, raw, {
detailLimit: 260, detailLimit: 260,
}); });
return applyAuthResponseCookies(response, auth.response); return finishProxyTimedResponse(
applyAuthResponseCookies(response, auth.response),
timer,
`upstream_${res.status}`,
{ backendServerTiming },
);
} }
const data = await res.json(); const data = await timer.measure("backend_read", () => res.json());
const response = NextResponse.json(data); const response = NextResponse.json(data);
return applyAuthResponseCookies(response, auth.response); return finishProxyTimedResponse(
applyAuthResponseCookies(response, auth.response),
timer,
"ok",
{ backendServerTiming },
);
} catch (error) { } catch (error) {
return buildProxyExceptionResponse(error, { const timedOut = controller.signal.aborted;
const response = timedOut
? NextResponse.json(
{
ok: false,
accepted: true,
dropped: true,
reason: "timeout",
},
{ status: 202 },
)
: buildProxyExceptionResponse(error, {
publicMessage: "Failed to track analytics event", publicMessage: "Failed to track analytics event",
}); });
const withCookies = auth ? applyAuthResponseCookies(response, auth.response) : response;
return finishProxyTimedResponse(
withCookies,
timer,
timedOut ? "timeout_accepted" : "exception",
);
} finally {
clearTimeout(timeoutId);
} }
} }
+19 -4
View File
@@ -1,6 +1,10 @@
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
import { proxyBackendJsonGet } from "@/lib/api-proxy"; import { proxyBackendJsonGet } from "@/lib/api-proxy";
import { buildCityDetailProxyCachePolicy } from "@/lib/proxy-cache-policy"; import { buildCityDetailProxyCachePolicy } from "@/lib/proxy-cache-policy";
import {
createProxyTimer,
finishProxyTimedResponse,
} from "@/lib/proxy-timing";
const API_BASE = process.env.POLYWEATHER_API_BASE_URL; const API_BASE = process.env.POLYWEATHER_API_BASE_URL;
const DETAIL_BATCH_PROXY_TIMEOUT_MS = Number( const DETAIL_BATCH_PROXY_TIMEOUT_MS = Number(
@@ -8,10 +12,15 @@ const DETAIL_BATCH_PROXY_TIMEOUT_MS = Number(
); );
export async function GET(req: NextRequest) { export async function GET(req: NextRequest) {
const timer = createProxyTimer(req, "city_detail_batch");
if (!API_BASE) { if (!API_BASE) {
return NextResponse.json( return finishProxyTimedResponse(
NextResponse.json(
{ error: "POLYWEATHER_API_BASE_URL is not configured" }, { error: "POLYWEATHER_API_BASE_URL is not configured" },
{ status: 500 }, { status: 500 },
),
timer,
"missing_api_base",
); );
} }
@@ -22,7 +31,7 @@ export async function GET(req: NextRequest) {
force_refresh: forceRefresh, force_refresh: forceRefresh,
limit: req.nextUrl.searchParams.get("limit") || "12", limit: req.nextUrl.searchParams.get("limit") || "12",
}); });
for (const key of ["market_slug", "target_date", "resolution"]) { for (const key of ["market_slug", "target_date", "resolution", "scope"]) {
const value = req.nextUrl.searchParams.get(key); const value = req.nextUrl.searchParams.get(key);
if (value) searchParams.set(key, value); if (value) searchParams.set(key, value);
} }
@@ -33,12 +42,18 @@ export async function GET(req: NextRequest) {
try { try {
return await proxyBackendJsonGet(req, { return await proxyBackendJsonGet(req, {
cacheControl: cachePolicy.responseCacheControl, cacheControl: cachePolicy.responseCacheControl,
fetchCache: cacheControlForData: (data) =>
cachePolicy.fetchMode === "no-store" ? "no-store" : undefined, data &&
typeof data === "object" &&
(data as { partial?: unknown }).partial === true
? "no-store, max-age=0"
: cachePolicy.responseCacheControl,
fetchCache: "no-store",
publicMessage: "Failed to fetch city detail batch", publicMessage: "Failed to fetch city detail batch",
revalidateSeconds: cachePolicy.revalidateSeconds, revalidateSeconds: cachePolicy.revalidateSeconds,
signal: controller.signal, signal: controller.signal,
timeoutPublicMessage: "City detail batch request timed out", timeoutPublicMessage: "City detail batch request timed out",
timing: timer,
url: `${API_BASE}/api/cities/detail-batch?${searchParams.toString()}`, url: `${API_BASE}/api/cities/detail-batch?${searchParams.toString()}`,
}); });
} finally { } finally {
+33 -11
View File
@@ -9,6 +9,10 @@ import {
} from "@/lib/api-proxy"; } from "@/lib/api-proxy";
import { buildCachedJsonResponse } from "@/lib/http-cache"; import { buildCachedJsonResponse } from "@/lib/http-cache";
import { buildCityDetailProxyCachePolicy } from "@/lib/proxy-cache-policy"; import { buildCityDetailProxyCachePolicy } from "@/lib/proxy-cache-policy";
import {
createProxyTimer,
finishProxyTimedResponse,
} from "@/lib/proxy-timing";
const API_BASE = process.env.POLYWEATHER_API_BASE_URL; const API_BASE = process.env.POLYWEATHER_API_BASE_URL;
@@ -34,15 +38,16 @@ export async function GET(
req: NextRequest, req: NextRequest,
context: { params: Promise<{ name: string }> }, context: { params: Promise<{ name: string }> },
) { ) {
const timer = createProxyTimer(req, "city_detail");
if (!API_BASE) { if (!API_BASE) {
const response = NextResponse.json( const response = NextResponse.json(
{ error: "POLYWEATHER_API_BASE_URL is not configured" }, { error: "POLYWEATHER_API_BASE_URL is not configured" },
{ status: 500 }, { status: 500 },
); );
return response; return finishProxyTimedResponse(response, timer, "missing_api_base");
} }
const { name } = await context.params; const { name } = await timer.measure("route_params", () => context.params);
const forceRefresh = req.nextUrl.searchParams.get("force_refresh") ?? "false"; const forceRefresh = req.nextUrl.searchParams.get("force_refresh") ?? "false";
const cachePolicy = buildCityDetailProxyCachePolicy(forceRefresh, 15); const cachePolicy = buildCityDetailProxyCachePolicy(forceRefresh, 15);
const depth = req.nextUrl.searchParams.get("depth"); const depth = req.nextUrl.searchParams.get("depth");
@@ -67,31 +72,48 @@ export async function GET(
const url = `${API_BASE}/api/city/${encodeURIComponent(name)}/detail?${searchParams.toString()}`; const url = `${API_BASE}/api/city/${encodeURIComponent(name)}/detail?${searchParams.toString()}`;
try { try {
const auth = await buildBackendRequestHeaders(req, { const auth = await timer.measure("auth_headers", () =>
buildBackendRequestHeaders(req, {
includeSupabaseIdentity: false, includeSupabaseIdentity: false,
}); }),
const res = await fetch(url, { );
const res = await timer.measure("backend_fetch", () =>
fetch(url, {
headers: auth.headers, headers: auth.headers,
...(cachePolicy.fetchMode === "no-store" ...(cachePolicy.fetchMode === "no-store"
? { cache: "no-store" as const } ? { cache: "no-store" as const }
: { next: { revalidate: cachePolicy.revalidateSeconds ?? 15 } }), : { next: { revalidate: cachePolicy.revalidateSeconds ?? 15 } }),
}); }),
);
const backendServerTiming = res.headers.get("server-timing") || "";
if (!res.ok) { if (!res.ok) {
const raw = await res.text(); const raw = await timer.measure("backend_read", () => res.text());
const response = buildUpstreamErrorResponse(res.status, raw); const response = buildUpstreamErrorResponse(res.status, raw);
return applyAuthResponseCookies(response, auth.response); return finishProxyTimedResponse(
applyAuthResponseCookies(response, auth.response),
timer,
`upstream_${res.status}`,
{ backendServerTiming },
);
} }
const data = normalizeCityDetailPayload(await res.json()); const data = normalizeCityDetailPayload(
await timer.measure("backend_read", () => res.json()),
);
const response = buildCachedJsonResponse( const response = buildCachedJsonResponse(
req, req,
data, data,
cachePolicy.responseCacheControl, cachePolicy.responseCacheControl,
); );
return applyAuthResponseCookies(response, auth.response); return finishProxyTimedResponse(
applyAuthResponseCookies(response, auth.response),
timer,
"ok",
{ backendServerTiming },
);
} catch (error) { } catch (error) {
const response = buildProxyExceptionResponse(error, { const response = buildProxyExceptionResponse(error, {
publicMessage: "Failed to fetch city detail aggregate", publicMessage: "Failed to fetch city detail aggregate",
}); });
return response; return finishProxyTimedResponse(response, timer, "exception");
} }
} }
+37 -10
View File
@@ -5,27 +5,45 @@ import {
} from "@/lib/backend-auth"; } from "@/lib/backend-auth";
import { buildProxyExceptionResponse } from "@/lib/api-proxy"; import { buildProxyExceptionResponse } from "@/lib/api-proxy";
import { requireOpsProxyAuth } from "@/lib/ops-proxy-auth"; import { requireOpsProxyAuth } from "@/lib/ops-proxy-auth";
import {
createProxyTimer,
finishProxyTimedResponse,
} from "@/lib/proxy-timing";
const API_BASE = process.env.POLYWEATHER_API_BASE_URL; const API_BASE = process.env.POLYWEATHER_API_BASE_URL;
export async function GET(req: NextRequest) { export async function GET(req: NextRequest) {
const timer = createProxyTimer(req, "ops_online_users");
if (!API_BASE) { if (!API_BASE) {
return NextResponse.json( return finishProxyTimedResponse(
NextResponse.json(
{ error: "POLYWEATHER_API_BASE_URL is not configured" }, { error: "POLYWEATHER_API_BASE_URL is not configured" },
{ status: 500 }, { status: 500 },
),
timer,
"missing_api_base",
); );
} }
try { try {
const auth = await buildBackendRequestHeaders(req); const auth = await timer.measure("auth_headers", () =>
const authError = requireOpsProxyAuth(req, auth); buildBackendRequestHeaders(req),
if (authError) return authError; );
const authError = timer.measureSync("ops_auth", () =>
requireOpsProxyAuth(req, auth),
);
if (authError) {
return finishProxyTimedResponse(authError, timer, "ops_auth_error");
}
const res = await fetch(`${API_BASE}/api/ops/online-users`, { const res = await timer.measure("backend_fetch", () =>
fetch(`${API_BASE}/api/ops/online-users`, {
headers: auth.headers, headers: auth.headers,
cache: "no-store", cache: "no-store",
}); }),
const raw = await res.text(); );
const backendServerTiming = res.headers.get("server-timing") || "";
const raw = await timer.measure("backend_read", () => res.text());
const response = new NextResponse(raw, { const response = new NextResponse(raw, {
status: res.status, status: res.status,
headers: { headers: {
@@ -33,10 +51,19 @@ export async function GET(req: NextRequest) {
"Cache-Control": "no-store", "Cache-Control": "no-store",
}, },
}); });
return applyAuthResponseCookies(response, auth.response); return finishProxyTimedResponse(
applyAuthResponseCookies(response, auth.response),
timer,
res.ok ? "ok" : `upstream_${res.status}`,
{ backendServerTiming },
);
} catch (error) { } catch (error) {
return buildProxyExceptionResponse(error, { return finishProxyTimedResponse(
buildProxyExceptionResponse(error, {
publicMessage: "Failed to fetch online users", publicMessage: "Failed to fetch online users",
}); }),
timer,
"exception",
);
} }
} }
+12 -2
View File
@@ -2,19 +2,28 @@ import { NextRequest, NextResponse } from "next/server";
import { proxyBackendJsonGet } from "@/lib/api-proxy"; import { proxyBackendJsonGet } from "@/lib/api-proxy";
import { buildForceRefreshProxyCachePolicy } from "@/lib/proxy-cache-policy"; import { buildForceRefreshProxyCachePolicy } from "@/lib/proxy-cache-policy";
import { DASHBOARD_REFRESH_POLICY_SEC } from "@/lib/refresh-policy"; import { DASHBOARD_REFRESH_POLICY_SEC } from "@/lib/refresh-policy";
import {
createProxyTimer,
finishProxyTimedResponse,
} from "@/lib/proxy-timing";
const API_BASE = process.env.POLYWEATHER_API_BASE_URL; const API_BASE = process.env.POLYWEATHER_API_BASE_URL;
const SCAN_TERMINAL_PROXY_TIMEOUT_MS = Number( const SCAN_TERMINAL_PROXY_TIMEOUT_MS = Number(
process.env.POLYWEATHER_SCAN_TERMINAL_PROXY_TIMEOUT_MS || "40000", process.env.POLYWEATHER_SCAN_TERMINAL_PROXY_TIMEOUT_MS || "18000",
); );
export const maxDuration = 45; export const maxDuration = 45;
export async function GET(req: NextRequest) { export async function GET(req: NextRequest) {
const timer = createProxyTimer(req, "scan_terminal");
if (!API_BASE) { if (!API_BASE) {
return NextResponse.json( return finishProxyTimedResponse(
NextResponse.json(
{ error: "POLYWEATHER_API_BASE_URL is not configured" }, { error: "POLYWEATHER_API_BASE_URL is not configured" },
{ status: 500 }, { status: 500 },
),
timer,
"missing_api_base",
); );
} }
@@ -61,6 +70,7 @@ export async function GET(req: NextRequest) {
revalidateSeconds: cachePolicy.revalidateSeconds, revalidateSeconds: cachePolicy.revalidateSeconds,
signal: controller.signal, signal: controller.signal,
timeoutPublicMessage: "Scan terminal request timed out", timeoutPublicMessage: "Scan terminal request timed out",
timing: timer,
url, url,
}); });
} finally { } finally {
@@ -46,7 +46,11 @@ import { scanRootClass } from "@/components/dashboard/scan-root-styles";
import { useRelativeTime } from "@/hooks/useRelativeTime"; import { useRelativeTime } from "@/hooks/useRelativeTime";
import { Panel } from "@/components/dashboard/scan-terminal/Panel"; import { Panel } from "@/components/dashboard/scan-terminal/Panel";
import { UsageGuideDashboard } from "@/components/dashboard/scan-terminal/UsageGuideDashboard"; import { UsageGuideDashboard } from "@/components/dashboard/scan-terminal/UsageGuideDashboard";
import { LiveTemperatureThresholdChart, clearCityDetailCache } from "@/components/dashboard/scan-terminal/LiveTemperatureThresholdChart"; import {
LiveTemperatureThresholdChart,
clearCityDetailCache,
preloadTemperatureChartCanvas,
} from "@/components/dashboard/scan-terminal/LiveTemperatureThresholdChart";
import { KoyfinRowsTable } from "@/components/dashboard/scan-terminal/KoyfinRowsTable"; import { KoyfinRowsTable } from "@/components/dashboard/scan-terminal/KoyfinRowsTable";
import { rowName, pct, money, temp, edgeClass } from "@/components/dashboard/scan-terminal/utils"; import { rowName, pct, money, temp, edgeClass } from "@/components/dashboard/scan-terminal/utils";
import { CitySelectorDropdown } from "@/components/dashboard/scan-terminal/CitySelectorDropdown"; import { CitySelectorDropdown } from "@/components/dashboard/scan-terminal/CitySelectorDropdown";
@@ -55,10 +59,15 @@ import {
mergeAccessStateWithAuthPayload, mergeAccessStateWithAuthPayload,
type AuthProfilePayload, type AuthProfilePayload,
} from "@/components/dashboard/scan-terminal/terminal-access-state"; } from "@/components/dashboard/scan-terminal/terminal-access-state";
import { loadTerminalAuthProfile } from "@/components/dashboard/scan-terminal/terminal-auth-bootstrap"; import {
createAuthProfileRequestCache,
loadTerminalAuthProfile,
} from "@/components/dashboard/scan-terminal/terminal-auth-bootstrap";
import { import {
cityListItemsToScanRows, cityListItemsToScanRows,
mergeScanRowsWithCityFallbackRows, mergeScanRowsWithCityFallbackRows,
readCachedCityList,
writeCachedCityList,
} from "@/components/dashboard/scan-terminal/city-fallback-rows"; } from "@/components/dashboard/scan-terminal/city-fallback-rows";
import { markAnalyticsOnce, trackAppEvent } from "@/lib/app-analytics"; import { markAnalyticsOnce, trackAppEvent } from "@/lib/app-analytics";
import { STATIC_CITY_LIST } from "@/lib/static-cities"; import { STATIC_CITY_LIST } from "@/lib/static-cities";
@@ -78,6 +87,10 @@ const TrainingDashboard = dynamic(
); );
const ONLINE_USERS_REFRESH_MS = 5 * 60_000; const ONLINE_USERS_REFRESH_MS = 5 * 60_000;
const AUTH_PROFILE_REQUEST_TIMEOUT_MS = 4500;
const AUTH_DECISION_RECOVERY_MS = 10_000;
const ACTIVE_ACCESS_CACHE_KEY = "polyweather_terminal_active_access_v1";
const ACTIVE_ACCESS_CACHE_TTL_MS = 6 * 60 * 60 * 1000;
function createEmptyAccess(loading = true): ProAccessState { function createEmptyAccess(loading = true): ProAccessState {
return { return {
@@ -109,6 +122,67 @@ function createLocalAccess(): ProAccessState {
}; };
} }
function isFutureAccessExpiry(value: string | null | undefined, now = Date.now()) {
if (!value) return true;
const ts = Date.parse(value);
return Number.isFinite(ts) && ts > now;
}
function readCachedActiveAccess(now = Date.now()): ProAccessState | null {
if (typeof window === "undefined") return null;
try {
const raw = window.localStorage.getItem(ACTIVE_ACCESS_CACHE_KEY);
if (!raw) return null;
const cached = JSON.parse(raw);
const access = cached?.access as Partial<ProAccessState> | undefined;
const ts = Number(cached?.ts || 0);
if (!access?.authenticated || !access.subscriptionActive) return null;
if (!ts || now - ts < 0 || now - ts > ACTIVE_ACCESS_CACHE_TTL_MS) return null;
if (!isFutureAccessExpiry(access.subscriptionTotalExpiresAt || access.subscriptionExpiresAt, now)) {
return null;
}
return {
loading: false,
authenticated: true,
userId: access.userId ?? null,
subscriptionActive: true,
subscriptionPlanCode: access.subscriptionPlanCode ?? null,
subscriptionExpiresAt: access.subscriptionExpiresAt ?? null,
subscriptionTotalExpiresAt:
access.subscriptionTotalExpiresAt ?? access.subscriptionExpiresAt ?? null,
subscriptionQueuedDays: Number(access.subscriptionQueuedDays ?? 0),
points: Number(access.points ?? 0),
error: null,
};
} catch {
return null;
}
}
function writeCachedActiveAccess(access: ProAccessState) {
if (typeof window === "undefined") return;
if (!access.authenticated || !access.subscriptionActive) return;
if (!isFutureAccessExpiry(access.subscriptionTotalExpiresAt || access.subscriptionExpiresAt)) return;
try {
window.localStorage.setItem(
ACTIVE_ACCESS_CACHE_KEY,
JSON.stringify({
ts: Date.now(),
access: {
authenticated: access.authenticated,
userId: access.userId,
subscriptionActive: access.subscriptionActive,
subscriptionPlanCode: access.subscriptionPlanCode,
subscriptionExpiresAt: access.subscriptionExpiresAt,
subscriptionTotalExpiresAt: access.subscriptionTotalExpiresAt,
subscriptionQueuedDays: access.subscriptionQueuedDays,
points: access.points,
},
}),
);
} catch {}
}
function createTransientAccess(error: unknown): ProAccessState { function createTransientAccess(error: unknown): ProAccessState {
return { return {
...createEmptyAccess(true), ...createEmptyAccess(true),
@@ -425,6 +499,11 @@ function PolyWeatherTerminal({
const [activeNavKey, setActiveNavKey] = useState<string>("thresholds"); const [activeNavKey, setActiveNavKey] = useState<string>("thresholds");
const [onlineCount, setOnlineCount] = useState<number | null>(null); const [onlineCount, setOnlineCount] = useState<number | null>(null);
useEffect(() => {
if (activeNavKey !== "thresholds") return;
void preloadTemperatureChartCanvas();
}, [activeNavKey]);
useEffect(() => { useEffect(() => {
const fetchOnline = () => { const fetchOnline = () => {
if (typeof document !== "undefined" && document.visibilityState === "hidden") return; if (typeof document !== "undefined" && document.visibilityState === "hidden") return;
@@ -957,12 +1036,79 @@ function PolyWeatherTerminal({
); );
} }
function AuthSyncRecoveryScreen({
isEn,
onRetry,
rootClassName,
retrying,
themeMode,
userLocalTime,
}: {
isEn: boolean;
onRetry: () => void;
rootClassName: string;
retrying: boolean;
themeMode: "dark" | "light";
userLocalTime: string;
}) {
return (
<div className={rootClassName}>
<div className={clsx("flex h-screen w-full bg-[#e9edf3] text-slate-950", themeMode === "light" && "light")}>
<aside className="w-[52px] bg-[#171d24]" />
<main className="flex flex-1 flex-col">
<header className="flex h-[52px] items-center justify-between border-b border-slate-200 bg-white px-4">
<Link href="/" className="flex items-center gap-2 hover:opacity-90">
<img src="/logo.png" alt="PolyWeather" className="h-7 w-auto object-contain" />
<span className="text-sm font-semibold tracking-tight text-slate-900">Terminal</span>
</Link>
<div className="font-mono text-sm text-slate-500">{userLocalTime}</div>
</header>
<section className="grid flex-1 place-items-center p-6">
<div className="w-full max-w-md rounded-[6px] border border-amber-200 bg-white p-6 shadow-sm">
<div className="mb-3 inline-flex rounded-full border border-amber-200 bg-amber-50 px-2.5 py-1 text-[11px] font-black text-amber-700">
{isEn ? "Access sync timeout" : "权限同步超时"}
</div>
<h1 className="text-xl font-black text-slate-950">
{isEn ? "We could not confirm your subscription yet" : "暂时未能确认你的订阅状态"}
</h1>
<p className="mt-3 text-sm leading-6 text-slate-600">
{isEn
? "The session or entitlement service is taking too long. Retry access sync; if you just paid, wait a few seconds and try again."
: "当前会话或会员服务响应过慢。请先重新同步权限;如果刚完成支付,等待几秒后再试。"}
</p>
<div className="mt-5 flex flex-col gap-3 sm:flex-row">
<button
type="button"
onClick={onRetry}
disabled={retrying}
className="inline-flex flex-1 items-center justify-center gap-2 rounded-[4px] bg-blue-600 px-4 py-2.5 text-sm font-black text-white transition hover:bg-blue-700 disabled:cursor-wait disabled:opacity-70"
>
{retrying && (
<span className="h-4 w-4 animate-spin rounded-full border-2 border-white/70 border-t-transparent" />
)}
{isEn ? "Retry access sync" : "重新同步权限"}
</button>
<Link
href="/account"
className="inline-flex flex-1 items-center justify-center rounded-[4px] border border-slate-200 bg-white px-4 py-2.5 text-sm font-black text-slate-700 transition hover:bg-slate-50"
>
{isEn ? "Open account" : "打开账户页"}
</Link>
</div>
</div>
</section>
</main>
</div>
</div>
);
}
function ScanTerminalScreen() { function ScanTerminalScreen() {
const [proAccess, setProAccess] = useState<ProAccessState>(() => const [proAccess, setProAccess] = useState<ProAccessState>(() =>
createEmptyAccess(true), createEmptyAccess(true),
); );
const loadAuthProfile = useCallback( const rawLoadAuthProfile = useCallback(
async ( async (
accessToken?: string | null, accessToken?: string | null,
options?: { preferSnapshot?: boolean }, options?: { preferSnapshot?: boolean },
@@ -970,6 +1116,11 @@ function ScanTerminalScreen() {
const headers: Record<string, string> = { Accept: "application/json" }; const headers: Record<string, string> = { Accept: "application/json" };
const token = String(accessToken || "").trim(); const token = String(accessToken || "").trim();
if (token) headers.Authorization = `Bearer ${token}`; if (token) headers.Authorization = `Bearer ${token}`;
const controller = typeof AbortController !== "undefined" ? new AbortController() : null;
const timeoutId = controller
? globalThis.setTimeout(() => controller.abort(), AUTH_PROFILE_REQUEST_TIMEOUT_MS)
: null;
try {
const response = await fetch( const response = await fetch(
options?.preferSnapshot options?.preferSnapshot
? "/api/auth/me?prefer_snapshot=1" ? "/api/auth/me?prefer_snapshot=1"
@@ -977,13 +1128,39 @@ function ScanTerminalScreen() {
{ {
cache: "no-store", cache: "no-store",
headers, headers,
signal: controller?.signal,
}, },
); );
if (!response.ok) throw new Error(`HTTP ${response.status}`); if (!response.ok) throw new Error(`HTTP ${response.status}`);
return response.json() as Promise<AuthProfilePayload>; return response.json() as Promise<AuthProfilePayload>;
} finally {
if (timeoutId !== null) globalThis.clearTimeout(timeoutId);
}
}, },
[], [],
); );
const authProfileRequestCacheRef = useRef<{
load: typeof rawLoadAuthProfile;
cached: ReturnType<typeof createAuthProfileRequestCache>;
} | null>(null);
const loadAuthProfile = useCallback(
(
accessToken?: string | null,
options?: { preferSnapshot?: boolean },
): Promise<AuthProfilePayload> => {
const current = authProfileRequestCacheRef.current;
if (current?.load === rawLoadAuthProfile) {
return current.cached(accessToken, options);
}
const next = {
load: rawLoadAuthProfile,
cached: createAuthProfileRequestCache(rawLoadAuthProfile),
};
authProfileRequestCacheRef.current = next;
return next.cached(accessToken, options);
},
[rawLoadAuthProfile],
);
const refreshLiveAuthProfile = useCallback(async () => { const refreshLiveAuthProfile = useCallback(async () => {
const supabaseEnabled = hasSupabasePublicEnv(); const supabaseEnabled = hasSupabasePublicEnv();
@@ -995,6 +1172,7 @@ function ScanTerminalScreen() {
hasSupabasePublicEnv: supabaseEnabled, hasSupabasePublicEnv: supabaseEnabled,
loadAuthProfile: (accessToken) => loadAuthProfile: (accessToken) =>
loadAuthProfile(accessToken, { preferSnapshot: false }), loadAuthProfile(accessToken, { preferSnapshot: false }),
timeoutMs: AUTH_PROFILE_REQUEST_TIMEOUT_MS + 2000,
}); });
setProAccess((prev) => mergeAccessStateWithAuthPayload(prev, payload)); setProAccess((prev) => mergeAccessStateWithAuthPayload(prev, payload));
}, [loadAuthProfile]); }, [loadAuthProfile]);
@@ -1057,13 +1235,17 @@ function ScanTerminalScreen() {
setLocale((prev) => (prev === "zh-CN" ? "en-US" : "zh-CN")); setLocale((prev) => (prev === "zh-CN" ? "en-US" : "zh-CN"));
const [hydrated, setHydrated] = useState(false); const [hydrated, setHydrated] = useState(false);
const [localFullAccess, setLocalFullAccess] = useState(false); const [localFullAccess, setLocalFullAccess] = useState(false);
const [authWaitExpired, setAuthWaitExpired] = useState(false);
const [authRetrying, setAuthRetrying] = useState(false);
const canUseLocalFullAccess = hydrated && localFullAccess; const canUseLocalFullAccess = hydrated && localFullAccess;
const isAuthenticated = const isAuthenticated =
hydrated && (proAccess.authenticated || canUseLocalFullAccess); hydrated && (proAccess.authenticated || canUseLocalFullAccess);
const isPro = const isPro =
hydrated && (proAccess.subscriptionActive || canUseLocalFullAccess); hydrated && (proAccess.subscriptionActive || canUseLocalFullAccess);
const accessDecisionPending = const accessDecisionPending =
!hydrated || (proAccess.loading && !canUseLocalFullAccess); !hydrated || (proAccess.loading && !canUseLocalFullAccess && !authWaitExpired);
const authSyncRecoveryNeeded =
hydrated && proAccess.loading && !canUseLocalFullAccess && authWaitExpired;
const shouldShowPaywall = !accessDecisionPending && (!isAuthenticated || !isPro); const shouldShowPaywall = !accessDecisionPending && (!isAuthenticated || !isPro);
const userLocalTime = useUserLocalClock(); const userLocalTime = useUserLocalClock();
const { themeMode } = useScanTerminalTheme(); const { themeMode } = useScanTerminalTheme();
@@ -1103,6 +1285,10 @@ function ScanTerminalScreen() {
cancelled = true; cancelled = true;
}; };
} }
const cachedActiveAccess = readCachedActiveAccess();
if (cachedActiveAccess) {
setProAccess(cachedActiveAccess);
}
if (typeof fetch !== "function") { if (typeof fetch !== "function") {
setProAccess(createEmptyAccess(false)); setProAccess(createEmptyAccess(false));
return () => { return () => {
@@ -1117,6 +1303,7 @@ function ScanTerminalScreen() {
: Promise.resolve({ data: { session: null } }), : Promise.resolve({ data: { session: null } }),
hasSupabasePublicEnv: supabaseEnabled, hasSupabasePublicEnv: supabaseEnabled,
loadAuthProfile, loadAuthProfile,
timeoutMs: AUTH_PROFILE_REQUEST_TIMEOUT_MS + 2000,
}) })
.then((payload) => { .then((payload) => {
if (cancelled) return; if (cancelled) return;
@@ -1140,6 +1327,23 @@ function ScanTerminalScreen() {
}; };
}, [loadAuthProfile, refreshLiveAuthProfile]); }, [loadAuthProfile, refreshLiveAuthProfile]);
useEffect(() => {
if (!hydrated || !proAccess.loading || canUseLocalFullAccess) {
setAuthWaitExpired(false);
return;
}
const timer = window.setTimeout(() => {
setAuthWaitExpired(true);
}, AUTH_DECISION_RECOVERY_MS);
return () => window.clearTimeout(timer);
}, [canUseLocalFullAccess, hydrated, proAccess.loading]);
useEffect(() => {
if (hydrated && proAccess.authenticated && proAccess.subscriptionActive) {
writeCachedActiveAccess(proAccess);
}
}, [hydrated, proAccess]);
useEffect(() => { useEffect(() => {
if ( if (
!hydrated || !hydrated ||
@@ -1163,6 +1367,7 @@ function ScanTerminalScreen() {
: Promise.resolve({ data: { session: null } }), : Promise.resolve({ data: { session: null } }),
hasSupabasePublicEnv: supabaseEnabled, hasSupabasePublicEnv: supabaseEnabled,
loadAuthProfile, loadAuthProfile,
timeoutMs: AUTH_PROFILE_REQUEST_TIMEOUT_MS + 2000,
}); });
if (cancelled) return; if (cancelled) return;
setProAccess((prev) => mergeAccessStateWithAuthPayload(prev, payload)); setProAccess((prev) => mergeAccessStateWithAuthPayload(prev, payload));
@@ -1228,9 +1433,20 @@ function ScanTerminalScreen() {
clearCityDetailCache(); clearCityDetailCache();
refreshScanTerminalManually(); refreshScanTerminalManually();
}, [refreshScanTerminalManually]); }, [refreshScanTerminalManually]);
const handleRetryAuthSync = useCallback(() => {
setAuthRetrying(true);
setAuthWaitExpired(false);
void refreshLiveAuthProfile()
.catch((error) => {
setProAccess((prev) => ({ ...prev, error: String(error) }));
})
.finally(() => {
setAuthRetrying(false);
});
}, [refreshLiveAuthProfile]);
const [cityFallbackRows, setCityFallbackRows] = useState<ScanOpportunityRow[]>(() => const [cityFallbackRows, setCityFallbackRows] = useState<ScanOpportunityRow[]>(() =>
cityListItemsToScanRows(STATIC_CITY_LIST), cityListItemsToScanRows(readCachedCityList() || STATIC_CITY_LIST),
); );
const rows = useMemo( const rows = useMemo(
() => { () => {
@@ -1246,9 +1462,15 @@ function ScanTerminalScreen() {
useEffect(() => { useEffect(() => {
if (!isPro || typeof fetch !== "function") return; if (!isPro || typeof fetch !== "function") return;
if (fallbackFetchedRef.current) return; if (fallbackFetchedRef.current) return;
const cachedCities = readCachedCityList();
if (cachedCities) {
fallbackFetchedRef.current = true;
setCityFallbackRows(cityListItemsToScanRows(cachedCities));
return;
}
const controller = new AbortController(); const controller = new AbortController();
fetch("/api/cities", { fetch("/api/cities", {
cache: "no-store", cache: "default",
headers: { Accept: "application/json" }, headers: { Accept: "application/json" },
signal: controller.signal, signal: controller.signal,
}) })
@@ -1259,6 +1481,7 @@ function ScanTerminalScreen() {
.then((payload) => { .then((payload) => {
if (!payload || !Array.isArray(payload.cities)) return; if (!payload || !Array.isArray(payload.cities)) return;
fallbackFetchedRef.current = true; fallbackFetchedRef.current = true;
writeCachedCityList(payload.cities);
setCityFallbackRows(cityListItemsToScanRows(payload.cities)); setCityFallbackRows(cityListItemsToScanRows(payload.cities));
}) })
.catch(() => {}); .catch(() => {});
@@ -1313,6 +1536,19 @@ function ScanTerminalScreen() {
); );
} }
if (authSyncRecoveryNeeded) {
return (
<AuthSyncRecoveryScreen
isEn={isEn}
onRetry={handleRetryAuthSync}
rootClassName={scanRootClass}
retrying={authRetrying}
themeMode={themeMode}
userLocalTime={userLocalTime}
/>
);
}
if (shouldShowPaywall) { if (shouldShowPaywall) {
return ( return (
<ProductAccessRequired <ProductAccessRequired
@@ -1,13 +1,12 @@
"use client"; "use client";
import clsx from "clsx"; import clsx from "clsx";
import dynamic from "next/dynamic";
import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import type { ScanOpportunityRow } from "@/lib/dashboard-types"; import type { ScanOpportunityRow } from "@/lib/dashboard-types";
import { useLatestPatch, useSseResyncVersion } from "@/hooks/use-sse-patches"; import { useLatestPatch, useSseResyncVersion } from "@/hooks/use-sse-patches";
import { Panel } from "@/components/dashboard/scan-terminal/Panel"; import { Panel } from "@/components/dashboard/scan-terminal/Panel";
import { ModelCurvesSummary } from "@/components/dashboard/scan-terminal/ModelCurvesSummary"; import { ModelCurvesSummary } from "@/components/dashboard/scan-terminal/ModelCurvesSummary";
import { TemperatureChartCanvasFallback } from "@/components/dashboard/scan-terminal/TemperatureChartCanvasFallback"; import { TemperatureChartCanvas } from "@/components/dashboard/scan-terminal/TemperatureChartCanvas";
import { TemperatureRunwayDetails } from "@/components/dashboard/scan-terminal/TemperatureRunwayDetails"; import { TemperatureRunwayDetails } from "@/components/dashboard/scan-terminal/TemperatureRunwayDetails";
import { TemperatureStatsBars } from "@/components/dashboard/scan-terminal/TemperatureStatsBars"; import { TemperatureStatsBars } from "@/components/dashboard/scan-terminal/TemperatureStatsBars";
import { rowName } from "@/components/dashboard/scan-terminal/utils"; import { rowName } from "@/components/dashboard/scan-terminal/utils";
@@ -56,17 +55,16 @@ const PEAK_GLOW_BADGE_CLASS = {
} as const; } as const;
const PROBABILITY_REFRESH_AFTER_PATCH_MS = 60_000; const PROBABILITY_REFRESH_AFTER_PATCH_MS = 60_000;
const FOREGROUND_FULL_DETAIL_REFRESH_DEDUP_MS = 90_000;
const DETAIL_LOAD_BATCH_DELAY_MS = 0;
const INITIAL_DETAIL_LOAD_SLOTS = 3;
const DEFERRED_DETAIL_LOAD_DELAY_MS = 1_200;
const DEFERRED_DETAIL_LOAD_GROUP_SIZE = 3;
const DEFERRED_DETAIL_LOAD_WAVE_STEP_MS = 900;
const TemperatureChartCanvas = dynamic( export function preloadTemperatureChartCanvas() {
() => return Promise.resolve();
import("@/components/dashboard/scan-terminal/TemperatureChartCanvas").then( }
(mod) => mod.TemperatureChartCanvas,
),
{
ssr: false,
loading: () => <TemperatureChartCanvasFallback />,
},
);
function peakGlowLabel(state: keyof typeof PEAK_GLOW_PANEL_CLASS, isEn: boolean) { function peakGlowLabel(state: keyof typeof PEAK_GLOW_PANEL_CLASS, isEn: boolean) {
if (state === "watch") return isEn ? "Watch" : "关注"; if (state === "watch") return isEn ? "Watch" : "关注";
@@ -113,12 +111,49 @@ function shouldFetchCityDetailForChart({
city, city,
documentHidden, documentHidden,
isChartVisible, isChartVisible,
compact = false,
isActive = false,
isMaximized = false,
slotIndex = 0,
detailLoadReady = true,
}: { }: {
city: string; city: string;
documentHidden: boolean; documentHidden: boolean;
isChartVisible: boolean; isChartVisible: boolean;
compact?: boolean;
isActive?: boolean;
isMaximized?: boolean;
slotIndex?: number;
detailLoadReady?: boolean;
}) { }) {
return Boolean(city) && isChartVisible && !documentHidden; if (!city || !isChartVisible || documentHidden) return false;
if (!compact || isActive || isMaximized) return true;
if (normalizeSlotIndex(slotIndex) < INITIAL_DETAIL_LOAD_SLOTS) return true;
return detailLoadReady;
}
function normalizeSlotIndex(slotIndex: number | null | undefined) {
return Number.isFinite(slotIndex) && Number(slotIndex) >= 0 ? Math.floor(Number(slotIndex)) : 0;
}
function getInitialDetailLoadDelayMs({
compact,
isActive,
isMaximized,
slotIndex,
}: {
compact?: boolean;
isActive?: boolean;
isMaximized?: boolean;
slotIndex?: number;
}) {
if (!compact || isActive || isMaximized) return 0;
const normalizedIndex = normalizeSlotIndex(slotIndex);
if (normalizedIndex < INITIAL_DETAIL_LOAD_SLOTS) return 0;
const deferredWave = Math.floor(
(normalizedIndex - INITIAL_DETAIL_LOAD_SLOTS) / DEFERRED_DETAIL_LOAD_GROUP_SIZE,
);
return DEFERRED_DETAIL_LOAD_DELAY_MS + deferredWave * DEFERRED_DETAIL_LOAD_WAVE_STEP_MS;
} }
// ── Main component ───────────────────────────────────────────────────── // ── Main component ─────────────────────────────────────────────────────
@@ -165,6 +200,7 @@ export function LiveTemperatureThresholdChart({
const lastPatchAtRef = useRef<number>(Date.now()); const lastPatchAtRef = useRef<number>(Date.now());
const lastAppliedPatchRevisionRef = useRef<number>(0); const lastAppliedPatchRevisionRef = useRef<number>(0);
const lastProbabilityRefreshAtRef = useRef<number>(0); const lastProbabilityRefreshAtRef = useRef<number>(0);
const lastForegroundRefreshAtRef = useRef<number>(0);
const localDayRolloverFetchDateRef = useRef<string>(""); const localDayRolloverFetchDateRef = useRef<string>("");
const [isChartVisible, setIsChartVisible] = useState( const [isChartVisible, setIsChartVisible] = useState(
() => typeof IntersectionObserver === "undefined", () => typeof IntersectionObserver === "undefined",
@@ -177,6 +213,11 @@ export function LiveTemperatureThresholdChart({
const [targetResolution, setTargetResolution] = useState<string>(() => const [targetResolution, setTargetResolution] = useState<string>(() =>
prefersHighFrequencyRunwayResolution(row, null) ? "1m" : "10m", prefersHighFrequencyRunwayResolution(row, null) ? "1m" : "10m",
); );
const detailLoadDelayMs = useMemo(
() => getInitialDetailLoadDelayMs({ compact, isActive, isMaximized, slotIndex }),
[compact, isActive, isMaximized, slotIndex],
);
const [detailLoadReady, setDetailLoadReady] = useState(() => detailLoadDelayMs === 0);
const [currentCityLocalDate, setCurrentCityLocalDate] = useState(() => const [currentCityLocalDate, setCurrentCityLocalDate] = useState(() =>
formatCityLocalDate(row?.tz_offset_seconds), formatCityLocalDate(row?.tz_offset_seconds),
); );
@@ -189,7 +230,7 @@ export function LiveTemperatureThresholdChart({
setTargetResolution(prefersHighFrequencyRunwayResolution(row, null) ? "1m" : "10m"); setTargetResolution(prefersHighFrequencyRunwayResolution(row, null) ? "1m" : "10m");
setHourly(seedHourlyForecastFromRow(row)); setHourly(seedHourlyForecastFromRow(row));
setLiveTemp(null); setLiveTemp(null);
setIsHourlyLoading(Boolean(city)); setIsHourlyLoading(Boolean(city) && detailLoadDelayMs === 0);
setDetailError(null); setDetailError(null);
setDetailRetryNonce(0); setDetailRetryNonce(0);
setShowingStaleDetail(false); setShowingStaleDetail(false);
@@ -197,9 +238,25 @@ export function LiveTemperatureThresholdChart({
lastPatchAtRef.current = Date.now(); lastPatchAtRef.current = Date.now();
lastAppliedPatchRevisionRef.current = 0; lastAppliedPatchRevisionRef.current = 0;
lastProbabilityRefreshAtRef.current = 0; lastProbabilityRefreshAtRef.current = 0;
lastForegroundRefreshAtRef.current = 0;
localDayRolloverFetchDateRef.current = ""; localDayRolloverFetchDateRef.current = "";
setCurrentCityLocalDate(formatCityLocalDate(row?.tz_offset_seconds)); setCurrentCityLocalDate(formatCityLocalDate(row?.tz_offset_seconds));
}, [city]); }, [city, detailLoadDelayMs]);
useEffect(() => {
if (!city) {
setDetailLoadReady(false);
return;
}
if (detailLoadDelayMs <= 0) {
setDetailLoadReady(true);
return;
}
setDetailLoadReady(false);
const id = setTimeout(() => setDetailLoadReady(true), detailLoadDelayMs);
return () => clearTimeout(id);
}, [city, detailLoadDelayMs]);
useEffect(() => { useEffect(() => {
const node = chartVisibilityRef.current; const node = chartVisibilityRef.current;
@@ -232,17 +289,6 @@ export function LiveTemperatureThresholdChart({
return; return;
} }
if (
!shouldFetchCityDetailForChart({
city,
documentHidden:
typeof document !== "undefined" && document.visibilityState === "hidden",
isChartVisible,
})
) {
return;
}
const cacheKey = `${city}:${targetResolution}`; const cacheKey = `${city}:${targetResolution}`;
let cached = _hourlyCache.get(cacheKey); let cached = _hourlyCache.get(cacheKey);
if (!cached || Date.now() - Number(cached.ts || 0) >= HOURLY_CACHE_TTL_MS) { if (!cached || Date.now() - Number(cached.ts || 0) >= HOURLY_CACHE_TTL_MS) {
@@ -267,6 +313,23 @@ export function LiveTemperatureThresholdChart({
return; return;
} }
if (
!shouldFetchCityDetailForChart({
city,
documentHidden:
typeof document !== "undefined" && document.visibilityState === "hidden",
isChartVisible,
compact,
isActive,
isMaximized,
slotIndex,
detailLoadReady,
})
) {
setIsHourlyLoading(false);
return;
}
if (!cached && !hasLoadedHourlyDetailRef.current) { if (!cached && !hasLoadedHourlyDetailRef.current) {
setHourly(seedHourlyForecastFromRow(row)); setHourly(seedHourlyForecastFromRow(row));
setShowingStaleDetail(false); setShowingStaleDetail(false);
@@ -274,9 +337,6 @@ export function LiveTemperatureThresholdChart({
setIsHourlyLoading(true); setIsHourlyLoading(true);
let cancelled = false; let cancelled = false;
// Prioritize active slots, stagger/delay background slots to optimize load performance
const delay = isActive ? 0 : (slotIndex ? 300 + slotIndex * 250 : 350);
const timer = setTimeout(() => { const timer = setTimeout(() => {
fetchHourlyForecastForCity(city, { resolution: targetResolution }) fetchHourlyForecastForCity(city, { resolution: targetResolution })
.then((data) => { .then((data) => {
@@ -296,13 +356,25 @@ export function LiveTemperatureThresholdChart({
.finally(() => { .finally(() => {
if (!cancelled) setIsHourlyLoading(false); if (!cancelled) setIsHourlyLoading(false);
}); });
}, delay); }, DETAIL_LOAD_BATCH_DELAY_MS);
return () => { return () => {
cancelled = true; cancelled = true;
clearTimeout(timer); clearTimeout(timer);
}; };
}, [city, row, isActive, slotIndex, targetResolution, isChartVisible, detailRetryNonce, isEn]); }, [
city,
row,
targetResolution,
isChartVisible,
compact,
isActive,
isMaximized,
slotIndex,
detailLoadReady,
detailRetryNonce,
isEn,
]);
useEffect(() => { useEffect(() => {
if (!latestPatch || latestPatch.revision <= lastAppliedPatchRevisionRef.current) return; if (!latestPatch || latestPatch.revision <= lastAppliedPatchRevisionRef.current) return;
@@ -398,7 +470,19 @@ export function LiveTemperatureThresholdChart({
let cancelled = false; let cancelled = false;
const refreshForegroundFullDetail = () => { const refreshForegroundFullDetail = () => {
lastPatchAtRef.current = Date.now(); const now = Date.now();
const cacheKey = `${city}:${targetResolution}`;
const cached = _hourlyCache.get(cacheKey);
const cacheAge = cached ? now - Number(cached.ts || 0) : Number.POSITIVE_INFINITY;
if (
now - lastForegroundRefreshAtRef.current < FOREGROUND_FULL_DETAIL_REFRESH_DEDUP_MS ||
(cacheAge >= 0 && cacheAge < FOREGROUND_FULL_DETAIL_REFRESH_DEDUP_MS)
) {
return;
}
lastForegroundRefreshAtRef.current = now;
lastPatchAtRef.current = now;
fetchHourlyForecastForCity(city, { ignoreCache: true, resolution: targetResolution }) fetchHourlyForecastForCity(city, { ignoreCache: true, resolution: targetResolution })
.then((data) => { .then((data) => {
@@ -885,6 +969,7 @@ export const __getLiveObservationLabelsForTest = getLiveObservationLabels;
export const __getObservationDisplayMetricsForTest = getObservationDisplayMetrics; export const __getObservationDisplayMetricsForTest = getObservationDisplayMetrics;
export const __getPeakGlowStateForTest = getPeakGlowState; export const __getPeakGlowStateForTest = getPeakGlowState;
export const __getWundergroundDailyHighForTest = getWundergroundDailyHigh; export const __getWundergroundDailyHighForTest = getWundergroundDailyHigh;
export const __getInitialDetailLoadDelayMsForTest = getInitialDetailLoadDelayMs;
export const __shouldFetchCityDetailForChartForTest = shouldFetchCityDetailForChart; export const __shouldFetchCityDetailForChartForTest = shouldFetchCityDetailForChart;
export const __shouldPollLiveChartForTest = shouldPollLiveChart; export const __shouldPollLiveChartForTest = shouldPollLiveChart;
export const __mergePatchIntoHourlyForTest = mergePatchIntoHourly; export const __mergePatchIntoHourlyForTest = mergePatchIntoHourly;
@@ -1,35 +0,0 @@
export function TemperatureChartCanvasFallback({ compact }: { compact?: boolean }) {
const horizontalLines = compact ? 5 : 7;
const verticalLines = compact ? 5 : 8;
const minChartHeight = compact === false ? 220 : 120;
return (
<div
className="relative flex-1 overflow-hidden rounded-sm border border-slate-100 bg-white"
style={{ minHeight: minChartHeight }}
>
<div className="absolute inset-x-3 bottom-7 top-4 rounded-sm border border-slate-100">
{Array.from({ length: horizontalLines }).map((_, index) => (
<span
key={`h-${index}`}
className="absolute left-0 right-0 border-t border-dashed border-sky-100"
style={{ top: `${(index / Math.max(1, horizontalLines - 1)) * 100}%` }}
/>
))}
{Array.from({ length: verticalLines }).map((_, index) => (
<span
key={`v-${index}`}
className="absolute bottom-0 top-0 border-l border-dashed border-sky-100"
style={{ left: `${(index / Math.max(1, verticalLines - 1)) * 100}%` }}
/>
))}
</div>
<div className="absolute inset-0 grid place-items-center">
<div className="flex items-center gap-2 rounded border border-slate-200 bg-white px-3 py-2 text-xs font-semibold text-slate-500 shadow-sm">
<span className="h-3 w-3 animate-spin rounded-full border-2 border-blue-200 border-t-blue-500" />
</div>
</div>
</div>
);
}
@@ -1,6 +1,9 @@
import { import {
CITY_LIST_CACHE_TTL_MS,
cityListItemsToScanRows, cityListItemsToScanRows,
mergeScanRowsWithCityFallbackRows, mergeScanRowsWithCityFallbackRows,
readCachedCityList,
writeCachedCityList,
} from "@/components/dashboard/scan-terminal/city-fallback-rows"; } from "@/components/dashboard/scan-terminal/city-fallback-rows";
import type { CityListItem } from "@/lib/dashboard-types"; import type { CityListItem } from "@/lib/dashboard-types";
@@ -47,6 +50,15 @@ export function runTests() {
]; ];
const rows = cityListItemsToScanRows(cities); const rows = cityListItemsToScanRows(cities);
const cacheStorage = new Map<string, string>();
const memoryStorage = {
getItem: (key: string) => cacheStorage.get(key) ?? null,
removeItem: (key: string) => { cacheStorage.delete(key); },
setItem: (key: string, value: string) => { cacheStorage.set(key, value); },
};
writeCachedCityList(cities, memoryStorage, 1_000);
const cachedCities = readCachedCityList(memoryStorage, 1_000 + 60_000);
assert(rows.length === 2, "fallback rows should preserve every city"); assert(rows.length === 2, "fallback rows should preserve every city");
assert(rows[0].id === "city-fallback:taipei", "fallback row id should be stable"); assert(rows[0].id === "city-fallback:taipei", "fallback row id should be stable");
@@ -57,6 +69,12 @@ export function runTests() {
assert(rows[0].temp_symbol === "°C", "celsius cities should use °C"); assert(rows[0].temp_symbol === "°C", "celsius cities should use °C");
assert(rows[1].trading_region === "north_america", "known cities should keep their configured product region"); assert(rows[1].trading_region === "north_america", "known cities should keep their configured product region");
assert(rows[1].temp_symbol === "°F", "fahrenheit cities should use °F"); assert(rows[1].temp_symbol === "°F", "fahrenheit cities should use °F");
assert(cachedCities?.length === 2, "fresh city registry cache should return the stored city list");
assert(cachedCities?.[0]?.name === "taipei", "city registry cache should preserve city payloads");
assert(
readCachedCityList(memoryStorage, 1_000 + CITY_LIST_CACHE_TTL_MS + 1) === null,
"expired city registry cache should be ignored so the dashboard can revalidate",
);
const scanRows = [ const scanRows = [
{ {
@@ -122,10 +140,13 @@ export function runTests() {
assert( assert(
dashboardSource.includes("cityListItemsToScanRows") && dashboardSource.includes("cityListItemsToScanRows") &&
dashboardSource.includes("STATIC_CITY_LIST") && dashboardSource.includes("STATIC_CITY_LIST") &&
dashboardSource.includes("readCachedCityList") &&
dashboardSource.includes("writeCachedCityList") &&
dashboardSource.includes("useState<ScanOpportunityRow[]>(() =>") && dashboardSource.includes("useState<ScanOpportunityRow[]>(() =>") &&
dashboardSource.includes("/api/cities") && dashboardSource.includes("/api/cities") &&
dashboardSource.includes('cache: "default"') &&
dashboardSource.includes("cityFallbackRows"), dashboardSource.includes("cityFallbackRows"),
"terminal dashboard should seed fallback rows from the static city snapshot before refreshing /api/cities", "terminal dashboard should seed fallback rows from cached/static city snapshots and avoid no-store /api/cities fetches when possible",
); );
assert(fs.existsSync(staticCitiesPath), "/api/cities route should have a static city snapshot fallback"); assert(fs.existsSync(staticCitiesPath), "/api/cities route should have a static city snapshot fallback");
assert( assert(
@@ -20,6 +20,7 @@ export function runTests() {
const cached = buildCityDetailProxyCachePolicy("false", 15); const cached = buildCityDetailProxyCachePolicy("false", 15);
assert.equal(cached.fetchMode, "revalidate"); assert.equal(cached.fetchMode, "revalidate");
assert.equal(cached.revalidateSeconds, 15); assert.equal(cached.revalidateSeconds, 15);
assert.match(cached.responseCacheControl, /max-age=15/);
assert.match(cached.responseCacheControl, /s-maxage=15/); assert.match(cached.responseCacheControl, /s-maxage=15/);
const scanForced = buildForceRefreshProxyCachePolicy("true", 10); const scanForced = buildForceRefreshProxyCachePolicy("true", 10);
@@ -55,6 +56,16 @@ export function runTests() {
/hasDirectBackendApiBaseUrl/, /hasDirectBackendApiBaseUrl/,
"public scan terminal requests should only attach user auth in direct-backend mode so CDN caches can be shared through the Next proxy", "public scan terminal requests should only attach user auth in direct-backend mode so CDN caches can be shared through the Next proxy",
); );
assert.match(
scanTerminalClientSource,
/SCAN_TERMINAL_PAYLOAD_VERSION/,
"scan terminal client should include a stable payload version in the URL so CDN caches roll forward after response-shape optimizations",
);
assert.match(
scanTerminalClientSource,
/params\.set\("_v",\s*SCAN_TERMINAL_PAYLOAD_VERSION\)/,
"scan terminal client should vary the CDN cache key without changing backend scan filters",
);
const overviewProxySource = fs.readFileSync( const overviewProxySource = fs.readFileSync(
path.join( path.join(
@@ -6,12 +6,14 @@ import {
} from "@/lib/refresh-policy"; } from "@/lib/refresh-policy";
import { scanTerminalQueryPolicy } from "@/components/dashboard/scan-terminal/scan-terminal-client"; import { scanTerminalQueryPolicy } from "@/components/dashboard/scan-terminal/scan-terminal-client";
import { import {
__getInitialDetailLoadDelayMsForTest,
__shouldFetchCityDetailForChartForTest, __shouldFetchCityDetailForChartForTest,
__shouldPollLiveChartForTest, __shouldPollLiveChartForTest,
} from "@/components/dashboard/scan-terminal/LiveTemperatureThresholdChart"; } from "@/components/dashboard/scan-terminal/LiveTemperatureThresholdChart";
import { import {
MAX_HOURLY_DETAIL_CONCURRENT_REQUESTS, MAX_HOURLY_DETAIL_CONCURRENT_REQUESTS,
HOURLY_CACHE_TTL_MS, HOURLY_CACHE_TTL_MS,
__resolveCityDetailFromBatchForTest,
__readHourlyCacheEntryForTest, __readHourlyCacheEntryForTest,
__resetHourlyDetailRequestQueueForTest, __resetHourlyDetailRequestQueueForTest,
__runQueuedHourlyDetailRequestForTest, __runQueuedHourlyDetailRequestForTest,
@@ -70,11 +72,33 @@ export async function runTests() {
chartSource.includes("2 * 60_000"), chartSource.includes("2 * 60_000"),
"selected city chart should consume SSE patches and use a 2-minute no-patch fallback", "selected city chart should consume SSE patches and use a 2-minute no-patch fallback",
); );
assert(
chartSource.includes("preloadTemperatureChartCanvas"),
"terminal chart canvas should expose a preload hook for first-paint optimization",
);
assert(
chartSource.includes('from "@/components/dashboard/scan-terminal/TemperatureChartCanvas"') &&
!chartSource.includes("next/dynamic") &&
!chartSource.includes("TemperatureChartCanvasFallback"),
"terminal chart canvas should be loaded with the terminal route instead of showing a second-stage dynamic chunk fallback",
);
assert(
dashboardSource.includes("preloadTemperatureChartCanvas") &&
dashboardSource.includes("void preloadTemperatureChartCanvas()") &&
dashboardSource.includes('activeNavKey !== "thresholds"'),
"terminal screen should preload the chart chunk once access is confirmed on the chart tab",
);
assert( assert(
chartSource.includes("fetchHourlyForecastForCity(city, { ignoreCache: true, resolution: targetResolution })") && chartSource.includes("fetchHourlyForecastForCity(city, { ignoreCache: true, resolution: targetResolution })") &&
chartSource.includes("setHourly(data)"), chartSource.includes("setHourly(data)"),
"visible chart fallback must refresh the full city detail payload at the current chart resolution when SSE patches stop", "visible chart fallback must refresh the full city detail payload at the current chart resolution when SSE patches stop",
); );
assert(
chartLogicSource.includes("HOURLY_FORCE_REFRESH_DEDUP_MS") &&
chartLogicSource.includes("maxAgeMs: HOURLY_FORCE_REFRESH_DEDUP_MS") &&
chartLogicSource.includes("recentlyRefreshed.data"),
"forced chart detail refreshes should reuse a very recent full-detail payload instead of refetching repeatedly",
);
assert( assert(
__shouldPollLiveChartForTest({ city: "shanghai", compact: true, isActive: false, isMaximized: false }) === true, __shouldPollLiveChartForTest({ city: "shanghai", compact: true, isActive: false, isMaximized: false }) === true,
"compact grid slots are visible charts and should run the no-patch fallback guard", "compact grid slots are visible charts and should run the no-patch fallback guard",
@@ -95,11 +119,38 @@ export async function runTests() {
chartLogicSource.includes("primeCityDetailCache"), chartLogicSource.includes("primeCityDetailCache"),
"visible terminal chart detail fetches should be coalesced into one batch request and prime the shared chart cache", "visible terminal chart detail fetches should be coalesced into one batch request and prime the shared chart cache",
); );
const fetchHourlyBlock = chartLogicSource.match(/async function fetchHourlyForecastForCity[\s\S]*?\n}\n\nfunction fetchCityDetailWithTimeout/)?.[0] || ""; assert(
chartLogicSource.includes('scope: "chart"') &&
chartLogicSource.includes("params.toString()"),
"terminal chart detail batches should request the slim chart scope instead of the full city detail payload",
);
assert(
chartLogicSource.includes("CITY_DETAIL_BATCH_WINDOW_MS = 100"),
"visible terminal chart detail fetches should use a wide enough batch window to coalesce cards mounted across adjacent frames",
);
assert(
chartLogicSource.includes("partial?: boolean") &&
chartLogicSource.includes("missing?: string[]"),
"frontend city detail batch payload should understand partial responses and missing city markers",
);
const flushCityDetailBatchBlock = chartLogicSource.match(/async function flushCityDetailBatch[\s\S]*?\r?\n}\r?\n\r?\nfunction fetchCityDetailBatchWithTimeout/)?.[0] || "";
assert(
flushCityDetailBatchBlock.includes("partialMissingCities") &&
flushCityDetailBatchBlock.includes("resolveBatchWaiters(waiters, null)") &&
flushCityDetailBatchBlock.includes("payload?.partial === true"),
"partial detail-batch misses should resolve without immediately issuing single-city fallback requests",
);
assert(
flushCityDetailBatchBlock.includes("if (!payload)") &&
flushCityDetailBatchBlock.includes("resolveAllBatchWaitersAsNull") &&
!flushCityDetailBatchBlock.includes("resolveCityDetailBatchWithSingleFallback"),
"whole-batch failures should stop at the chart batch layer instead of fanning out into single-city full-detail requests",
);
const fetchHourlyBlock = chartLogicSource.match(/async function fetchHourlyForecastForCity[\s\S]*?\r?\n}\r?\n\r?\nfunction shouldPollLiveChart/)?.[0] || "";
assert( assert(
fetchHourlyBlock.includes("queueCityDetailBatch(city, resParam)") && fetchHourlyBlock.includes("queueCityDetailBatch(city, resParam)") &&
!fetchHourlyBlock.includes("runQueuedHourlyDetailRequest"), !fetchHourlyBlock.includes("runQueuedHourlyDetailRequest"),
"first-paint and background city detail refreshes should both enter the batch queue before falling back to single requests", "first-paint and background city detail refreshes should both enter the batch queue without falling back to single requests",
); );
assert( assert(
dashboardSource.includes("ONLINE_USERS_REFRESH_MS = 5 * 60_000") && dashboardSource.includes("ONLINE_USERS_REFRESH_MS = 5 * 60_000") &&
@@ -112,6 +163,11 @@ export async function runTests() {
chartSource.includes("isChartVisible"), chartSource.includes("isChartVisible"),
"city detail prefetch should be gated to visible chart cards instead of every mounted slot", "city detail prefetch should be gated to visible chart cards instead of every mounted slot",
); );
assert(
chartSource.includes("DETAIL_LOAD_BATCH_DELAY_MS") &&
!chartSource.includes("slotIndex ? 300 + slotIndex * 250"),
"visible chart detail loads should enter the batch queue together instead of being staggered into single-city requests",
);
assert( assert(
chartSource.includes("allowStale: true") && chartSource.includes("allowStale: true") &&
chartCanvasSourceIncludes(chartSource, "数据暂不可用") && chartCanvasSourceIncludes(chartSource, "数据暂不可用") &&
@@ -130,6 +186,71 @@ export async function runTests() {
__shouldFetchCityDetailForChartForTest({ city: "paris", documentHidden: true, isChartVisible: true }) === false, __shouldFetchCityDetailForChartForTest({ city: "paris", documentHidden: true, isChartVisible: true }) === false,
"hidden browser tabs should not prefetch city detail", "hidden browser tabs should not prefetch city detail",
); );
assert(
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 0 }) === 0 &&
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 2 }) === 0,
"first visible grid charts should fetch full detail immediately",
);
assert(
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 3 }) > 0,
"later non-active grid charts should defer full-detail loading after first paint",
);
assert(
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 3 }) ===
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 5 }),
"deferred grid charts should load in same-wave groups so detail-batch can coalesce them",
);
assert(
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 6 }) ===
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 8 }) &&
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 6 }) >
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: false, slotIndex: 5 }),
"later deferred grid charts should form a second batch wave instead of one request per slot",
);
assert(
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: true, isMaximized: false, slotIndex: 8 }) === 0 &&
__getInitialDetailLoadDelayMsForTest({ compact: true, isActive: false, isMaximized: true, slotIndex: 8 }) === 0,
"active or maximized charts should bypass deferred detail loading",
);
assert(
__shouldFetchCityDetailForChartForTest({
city: "paris",
documentHidden: false,
isChartVisible: true,
compact: true,
isActive: false,
isMaximized: false,
slotIndex: 5,
detailLoadReady: false,
}) === false,
"deferred grid charts should not enter the detail request queue before their delay is ready",
);
assert(
__shouldFetchCityDetailForChartForTest({
city: "paris",
documentHidden: false,
isChartVisible: true,
compact: true,
isActive: false,
isMaximized: false,
slotIndex: 5,
detailLoadReady: true,
}) === true,
"deferred grid charts should fetch detail after their delay is ready",
);
const normalizedBatchDetail = __resolveCityDetailFromBatchForTest(
{
"hong kong": {
city: "hong kong",
timeseries: { hourly: { times: ["00:00"], temps: [32] } },
},
} as any,
"Hong Kong",
) as any;
assert(
normalizedBatchDetail?.city === "hong kong",
"frontend detail batch lookup should accept backend-normalized city keys before falling back to single-city requests",
);
__resetHourlyDetailRequestQueueForTest(); __resetHourlyDetailRequestQueueForTest();
let activeRequests = 0; let activeRequests = 0;
@@ -94,11 +94,30 @@ export function runTests() {
assert(hook.includes("subscribedCities"), "frontend patch hook must track the visible city subscription set"); assert(hook.includes("subscribedCities"), "frontend patch hook must track the visible city subscription set");
assert(hook.includes("since_revision"), "frontend patch hook must reconnect with since_revision"); assert(hook.includes("since_revision"), "frontend patch hook must reconnect with since_revision");
assert(hook.includes("resync_required"), "frontend patch hook must react to server resync_required events"); assert(hook.includes("resync_required"), "frontend patch hook must react to server resync_required events");
assert(
hook.includes("SSE_REPLAY_EVENTS_PER_CITY") &&
hook.includes("resolveSseReplayLimit") &&
hook.includes('params.set("replay_limit", String(resolveSseReplayLimit(cities.length)))') &&
!hook.includes('params.set("replay_limit", "500")'),
"frontend patch hook should size SSE replay_limit by visible city count instead of always asking for 500 events",
);
assert(hook.includes("lastRevision"), "frontend patch hook must track the global last processed revision"); assert(hook.includes("lastRevision"), "frontend patch hook must track the global last processed revision");
assert(hook.includes("Map<"), "frontend patch hook must keep latest patches in a Map"); assert(hook.includes("Map<"), "frontend patch hook must keep latest patches in a Map");
assert(hook.includes("useLatestPatch"), "frontend patch hook must export useLatestPatch(city)"); assert(hook.includes("useLatestPatch"), "frontend patch hook must export useLatestPatch(city)");
assert(hook.includes("revision"), "frontend patch hook must track revisions and skip stale patches"); assert(hook.includes("revision"), "frontend patch hook must track revisions and skip stale patches");
assert(hook.includes("setTimeout"), "frontend patch hook must implement explicit reconnect backoff"); assert(hook.includes("setTimeout"), "frontend patch hook must implement explicit reconnect backoff");
assert(
hook.includes("SSE_SUBSCRIPTION_RECONNECT_DELAY_MS") &&
hook.includes("scheduleSubscriptionReconnect") &&
hook.includes("clearSubscriptionReconnectTimer"),
"frontend patch hook must debounce visible-city subscription changes before reopening SSE",
);
const subscriptionBlock = hook.match(/function registerCitySubscription[\s\S]*?\n}\r?\n\r?\nfunction normalizeLegacyPatch/)?.[0] || "";
assert(
subscriptionBlock.includes("scheduleSubscriptionReconnect()") &&
!subscriptionBlock.includes("ensureSsePatchConnection();"),
"city subscription mount/unmount should schedule one coalesced SSE reconnect instead of reconnecting per chart",
);
const bffEventsRoute = readFrontendFile("app", "api", "events", "route.ts"); const bffEventsRoute = readFrontendFile("app", "api", "events", "route.ts");
assert(bffEventsRoute.includes("searchParams"), "Next.js SSE proxy must forward query parameters to FastAPI"); assert(bffEventsRoute.includes("searchParams"), "Next.js SSE proxy must forward query parameters to FastAPI");
@@ -179,8 +198,9 @@ export function runTests() {
assert( assert(
foregroundRefreshBlock.includes("ignoreCache: true") && foregroundRefreshBlock.includes("ignoreCache: true") &&
foregroundRefreshBlock.includes("fetchHourlyForecastForCity") && foregroundRefreshBlock.includes("fetchHourlyForecastForCity") &&
foregroundRefreshBlock.includes("FOREGROUND_FULL_DETAIL_REFRESH_DEDUP_MS") &&
!foregroundRefreshBlock.includes("setIsHourlyLoading(true)"), !foregroundRefreshBlock.includes("setIsHourlyLoading(true)"),
"foreground resume refresh should update full detail immediately in the background without showing the loading overlay", "foreground resume refresh should update full detail in the background without showing the loading overlay or refetching fresh detail",
); );
assert( assert(
!chart.includes("/api/city/${encodeURIComponent(city)}/summary"), !chart.includes("/api/city/${encodeURIComponent(city)}/summary"),
@@ -239,7 +259,7 @@ export function runTests() {
); );
assert( assert(
chartLogic.includes("HOURLY_DETAIL_REQUEST_TIMEOUT_MS = 12_000") && chartLogic.includes("HOURLY_DETAIL_REQUEST_TIMEOUT_MS = 12_000") &&
chartLogic.includes("fetchCityDetailWithTimeout") && chartLogic.includes("fetchCityDetailBatchWithTimeout") &&
chartLogic.includes("signal: controller.signal") && chartLogic.includes("signal: controller.signal") &&
chartLogic.includes("controller.abort()"), chartLogic.includes("controller.abort()"),
"city detail chart fetches must have a frontend timeout so panels cannot stay on 加载图表 forever", "city detail chart fetches must have a frontend timeout so panels cannot stay on 加载图表 forever",
@@ -1,4 +1,5 @@
import { import {
createAuthProfileRequestCache,
loadTerminalAuthProfile, loadTerminalAuthProfile,
type TerminalAuthProfilePayload, type TerminalAuthProfilePayload,
} from "@/components/dashboard/scan-terminal/terminal-auth-bootstrap"; } from "@/components/dashboard/scan-terminal/terminal-auth-bootstrap";
@@ -27,6 +28,41 @@ async function flushMicrotasks() {
} }
export async function runTests() { export async function runTests() {
const pendingProfile = deferred<TerminalAuthProfilePayload>();
let underlyingProfileLoads = 0;
const cachedLoadAuthProfile = createAuthProfileRequestCache((
accessToken?: string | null,
options?: { preferSnapshot?: boolean },
) => {
underlyingProfileLoads += 1;
assert(
accessToken === "shared-token" && options?.preferSnapshot === true,
"auth profile request cache should pass through the original token and snapshot preference",
);
return pendingProfile.promise;
});
const firstCachedProfile = cachedLoadAuthProfile("shared-token", {
preferSnapshot: true,
});
const secondCachedProfile = cachedLoadAuthProfile("shared-token", {
preferSnapshot: true,
});
assert(
firstCachedProfile === secondCachedProfile && underlyingProfileLoads === 1,
"auth profile request cache should dedupe identical in-flight profile loads",
);
pendingProfile.resolve({
authenticated: true,
user_id: "shared-user",
subscription_active: true,
});
await firstCachedProfile;
await cachedLoadAuthProfile("shared-token", { preferSnapshot: true });
assert(
underlyingProfileLoads === 2,
"auth profile request cache should clear a key after the in-flight load settles",
);
const slowCookieProfile = deferred<TerminalAuthProfilePayload>(); const slowCookieProfile = deferred<TerminalAuthProfilePayload>();
const fastSession = deferred<{ data: { session: { access_token: string } } }>(); const fastSession = deferred<{ data: { session: { access_token: string } } }>();
const calls: string[] = []; const calls: string[] = [];
@@ -88,6 +124,45 @@ export async function runTests() {
"terminal auth bootstrap should accept an authenticated cookie profile immediately", "terminal auth bootstrap should accept an authenticated cookie profile immediately",
); );
const slowDegradedSession = deferred<{ data: { session: { access_token: string } | null } }>();
const timedCookieResult = await loadTerminalAuthProfile({
hasSupabasePublicEnv: true,
getSession: () => slowDegradedSession.promise,
timeoutMs: 1,
loadAuthProfile: (accessToken) => {
assert(!accessToken, "timeout fallback should use the settled cookie profile when bearer session is still pending");
return Promise.resolve({
authenticated: true,
user_id: "cookie-user",
subscription_active: null,
degraded_auth_profile: true,
});
},
});
assert(
timedCookieResult.user_id === "cookie-user" &&
timedCookieResult.subscription_active === null,
"terminal auth bootstrap timeout must resolve with a known degraded cookie profile instead of spinning forever",
);
const neverCookie = deferred<TerminalAuthProfilePayload>();
const neverSession = deferred<{ data: { session: { access_token: string } | null } }>();
let timeoutRejected = false;
try {
await loadTerminalAuthProfile({
hasSupabasePublicEnv: true,
getSession: () => neverSession.promise,
timeoutMs: 1,
loadAuthProfile: () => neverCookie.promise,
});
} catch (error) {
timeoutRejected = String(error).includes("Terminal auth bootstrap timeout");
}
assert(
timeoutRejected,
"terminal auth bootstrap must reject instead of leaving the loading screen unresolved when no profile request settles",
);
const delayedBearerSession = deferred<{ data: { session: { access_token: string } } }>(); const delayedBearerSession = deferred<{ data: { session: { access_token: string } } }>();
let coldStartSettled = false; let coldStartSettled = false;
const coldStartResultPromise = loadTerminalAuthProfile({ const coldStartResultPromise = loadTerminalAuthProfile({
@@ -97,11 +97,10 @@ export function runTests() {
"terminal dashboard must lazy-load the training analytics tab so Recharts stays out of the default terminal path", "terminal dashboard must lazy-load the training analytics tab so Recharts stays out of the default terminal path",
); );
assert( assert(
chartSource.includes('from "next/dynamic"') && chartSource.includes('import { TemperatureChartCanvas } from "@/components/dashboard/scan-terminal/TemperatureChartCanvas";') &&
chartSource.includes("TemperatureChartCanvasFallback") && !chartSource.includes("TemperatureChartCanvasFallback") &&
chartSource.includes("import(\"@/components/dashboard/scan-terminal/TemperatureChartCanvas\")") && !chartSource.includes("import(\"@/components/dashboard/scan-terminal/TemperatureChartCanvas\")"),
!chartSource.includes('import { TemperatureChartCanvas } from "@/components/dashboard/scan-terminal/TemperatureChartCanvas";'), "terminal temperature charts must load the Recharts canvas with the terminal route so visible cards do not sit behind a second-stage fallback",
"terminal temperature charts must lazy-load the Recharts canvas behind a lightweight fallback",
); );
assert( assert(
chartSource.includes("setLiveTemp(null);") && chartSource.includes("setLiveTemp(null);") &&
@@ -5,10 +5,61 @@ import {
type RegionKey, type RegionKey,
} from "@/components/dashboard/scan-terminal/continent-grouping"; } from "@/components/dashboard/scan-terminal/continent-grouping";
const CITY_LIST_CACHE_KEY = "polyweather_city_list_v1";
export const CITY_LIST_CACHE_TTL_MS = 24 * 60 * 60 * 1000;
type CityListCacheStorage = Pick<Storage, "getItem" | "removeItem" | "setItem">;
function normalizeCityKey(value: string) { function normalizeCityKey(value: string) {
return String(value || "").trim().toLowerCase(); return String(value || "").trim().toLowerCase();
} }
function getCityListCacheStorage(storage?: CityListCacheStorage): CityListCacheStorage | null {
if (storage) return storage;
if (typeof window === "undefined") return null;
try {
return window.localStorage;
} catch {
return null;
}
}
export function readCachedCityList(
storage?: CityListCacheStorage,
now = Date.now(),
): CityListItem[] | null {
const cacheStorage = getCityListCacheStorage(storage);
if (!cacheStorage) return null;
try {
const raw = cacheStorage.getItem(CITY_LIST_CACHE_KEY);
if (!raw) return null;
const cached = JSON.parse(raw);
const age = now - Number(cached.ts || 0);
if (age < 0 || age >= CITY_LIST_CACHE_TTL_MS || !Array.isArray(cached.cities)) {
return null;
}
return cached.cities as CityListItem[];
} catch {
try { cacheStorage.removeItem(CITY_LIST_CACHE_KEY); } catch {}
}
return null;
}
export function writeCachedCityList(
cities: CityListItem[],
storage?: CityListCacheStorage,
now = Date.now(),
) {
const cacheStorage = getCityListCacheStorage(storage);
if (!cacheStorage || !Array.isArray(cities) || !cities.length) return;
try {
cacheStorage.setItem(
CITY_LIST_CACHE_KEY,
JSON.stringify({ ts: now, cities }),
);
} catch {}
}
function tempSymbolForCity(city: CityListItem) { function tempSymbolForCity(city: CityListItem) {
return city.temp_unit === "fahrenheit" ? "°F" : "°C"; return city.temp_unit === "fahrenheit" ? "°F" : "°C";
} }
@@ -22,6 +22,7 @@ export const scanTerminalQueryPolicy = {
autoRefreshMs: null, autoRefreshMs: null,
manualForceRefreshCooldownMs: 2 * 60_000, manualForceRefreshCooldownMs: 2 * 60_000,
} as const; } as const;
const SCAN_TERMINAL_PAYLOAD_VERSION = "runway-slim-v1";
type TerminalQueryOptions = { type TerminalQueryOptions = {
forceRefresh?: boolean; forceRefresh?: boolean;
@@ -142,6 +143,7 @@ async function getTerminal({
if (Number.isFinite(timezoneOffsetSeconds)) { if (Number.isFinite(timezoneOffsetSeconds)) {
params.set("timezone_offset_seconds", String(Math.trunc(Number(timezoneOffsetSeconds)))); params.set("timezone_offset_seconds", String(Math.trunc(Number(timezoneOffsetSeconds))));
} }
params.set("_v", SCAN_TERMINAL_PAYLOAD_VERSION);
if (forceRefresh) { if (forceRefresh) {
params.set("_ts", String(Date.now())); params.set("_ts", String(Date.now()));
} }
@@ -353,6 +353,7 @@ type LocalDayBounds = { start: number; end: number };
const MAX_OBS_POINTS = 1440; const MAX_OBS_POINTS = 1440;
const HOURLY_CACHE_TTL_MS = DASHBOARD_REFRESH_POLICY_MS.metar; const HOURLY_CACHE_TTL_MS = DASHBOARD_REFRESH_POLICY_MS.metar;
const HOURLY_FORCE_REFRESH_DEDUP_MS = 60_000;
const _hourlyCache = new Map<string, { ts: number; data: HourlyForecast }>(); const _hourlyCache = new Map<string, { ts: number; data: HourlyForecast }>();
const _hourlyRequestCache = new Map<string, Promise<HourlyForecast>>(); const _hourlyRequestCache = new Map<string, Promise<HourlyForecast>>();
const MAX_HOURLY_DETAIL_CONCURRENT_REQUESTS = 3; const MAX_HOURLY_DETAIL_CONCURRENT_REQUESTS = 3;
@@ -366,13 +367,16 @@ const SESSION_CACHE_TTL_MS = DASHBOARD_REFRESH_POLICY_MS.metar;
type HourlyCacheEntry = { ts: number; data: HourlyForecast }; type HourlyCacheEntry = { ts: number; data: HourlyForecast };
function isFreshHourlyCacheEntry(entry: HourlyCacheEntry | null | undefined) { function isFreshHourlyCacheEntry(
return Boolean(entry && Date.now() - Number(entry.ts || 0) < SESSION_CACHE_TTL_MS); entry: HourlyCacheEntry | null | undefined,
maxAgeMs = SESSION_CACHE_TTL_MS,
) {
return Boolean(entry && Date.now() - Number(entry.ts || 0) < maxAgeMs);
} }
function readSessionCache( function readSessionCache(
city: string, city: string,
options: { allowStale?: boolean } = {}, options: { allowStale?: boolean; maxAgeMs?: number } = {},
): HourlyCacheEntry | null { ): HourlyCacheEntry | null {
if (typeof window === "undefined") return null; if (typeof window === "undefined") return null;
try { try {
@@ -382,7 +386,7 @@ function readSessionCache(
if ( if (
item && item &&
item.ts && item.ts &&
(options.allowStale || Date.now() - item.ts < SESSION_CACHE_TTL_MS) (options.allowStale || Date.now() - item.ts < (options.maxAgeMs ?? SESSION_CACHE_TTL_MS))
) { ) {
return item; return item;
} }
@@ -392,10 +396,10 @@ function readSessionCache(
function readHourlyCacheEntry( function readHourlyCacheEntry(
cacheKey: string, cacheKey: string,
options: { allowStale?: boolean } = {}, options: { allowStale?: boolean; maxAgeMs?: number } = {},
): HourlyCacheEntry | null { ): HourlyCacheEntry | null {
const cached = _hourlyCache.get(cacheKey); const cached = _hourlyCache.get(cacheKey);
if (cached && (options.allowStale || isFreshHourlyCacheEntry(cached))) { if (cached && (options.allowStale || isFreshHourlyCacheEntry(cached, options.maxAgeMs))) {
return cached; return cached;
} }
@@ -966,8 +970,11 @@ type HourlyForecastFetchOptions = {
}; };
type CityDetailBatchPayload = { type CityDetailBatchPayload = {
cities?: string[];
details?: Record<string, CityDetail | null | undefined>; details?: Record<string, CityDetail | null | undefined>;
errors?: Record<string, string>; errors?: Record<string, string>;
missing?: string[];
partial?: boolean;
}; };
type CityDetailBatchWaiter = { type CityDetailBatchWaiter = {
@@ -981,7 +988,7 @@ type CityDetailBatchQueue = {
timer: ReturnType<typeof setTimeout> | null; timer: ReturnType<typeof setTimeout> | null;
}; };
const CITY_DETAIL_BATCH_WINDOW_MS = 25; const CITY_DETAIL_BATCH_WINDOW_MS = 100;
const CITY_DETAIL_BATCH_MAX_CITIES = 12; const CITY_DETAIL_BATCH_MAX_CITIES = 12;
const _cityDetailBatchQueues = new Map<string, CityDetailBatchQueue>(); const _cityDetailBatchQueues = new Map<string, CityDetailBatchQueue>();
@@ -1026,16 +1033,6 @@ function primeCityDetailCache(
return data; return data;
} }
async function fetchSingleHourlyForecastForCity(
city: string,
resolution: string,
): Promise<HourlyForecast> {
const res = await fetchCityDetailWithTimeout(city, resolution);
if (!res || !res.ok) return null;
const json = await res.json() as CityDetail;
return primeCityDetailCache(city, resolution, json);
}
function queueCityDetailBatch(city: string, resolution: string): Promise<HourlyForecast> { function queueCityDetailBatch(city: string, resolution: string): Promise<HourlyForecast> {
return new Promise<HourlyForecast>((resolve, reject) => { return new Promise<HourlyForecast>((resolve, reject) => {
const queue = _cityDetailBatchQueues.get(resolution) || { const queue = _cityDetailBatchQueues.get(resolution) || {
@@ -1066,11 +1063,28 @@ function resolveBatchWaiters(
(waiters || []).forEach((waiter) => waiter.resolve(value)); (waiters || []).forEach((waiter) => waiter.resolve(value));
} }
function rejectBatchWaiters( function resolveCityDetailFromBatch(
waiters: CityDetailBatchWaiter[] | undefined, details: Record<string, CityDetail | null | undefined> | undefined,
reason: unknown, city: string,
) { ) {
(waiters || []).forEach((waiter) => waiter.reject(reason)); if (!details) return undefined;
const trimmed = String(city || "").trim();
const direct =
details[city] ||
details[trimmed] ||
details[trimmed.toLowerCase()] ||
details[normalizeCityKey(trimmed)];
if (direct) return direct;
const requestedKey = normalizeCityKey(trimmed);
if (!requestedKey) return undefined;
for (const [key, detail] of Object.entries(details)) {
if (!detail) continue;
if (normalizeCityKey(key) === requestedKey) return detail;
const detailCity = (detail as any).city || detail.name || detail.display_name;
if (normalizeCityKey(detailCity) === requestedKey) return detail;
}
return undefined;
} }
async function flushCityDetailBatch(resolution: string) { async function flushCityDetailBatch(resolution: string) {
@@ -1087,41 +1101,44 @@ async function flushCityDetailBatch(resolution: string) {
try { try {
const payload = await fetchCityDetailBatchWithTimeout(cities, resolution); const payload = await fetchCityDetailBatchWithTimeout(cities, resolution);
if (!payload) {
resolveAllBatchWaitersAsNull(cities, queue);
return;
}
const details = payload?.details || {}; const details = payload?.details || {};
const partialMissingCities =
payload?.partial === true
? new Set((payload.missing || []).map((city) => normalizeCityKey(city)))
: new Set<string>();
await Promise.all( await Promise.all(
cities.map(async (city) => { cities.map(async (city) => {
const waiters = queue.waiters.get(city); const waiters = queue.waiters.get(city);
const detail = details[city]; const detail = resolveCityDetailFromBatch(details, city);
const data = primeCityDetailCache(city, resolution, detail); const data = primeCityDetailCache(city, resolution, detail);
if (data) { if (data) {
resolveBatchWaiters(waiters, data); resolveBatchWaiters(waiters, data);
return; return;
} }
try { if (partialMissingCities.has(normalizeCityKey(city))) {
resolveBatchWaiters( resolveBatchWaiters(waiters, null);
waiters, return;
await runQueuedHourlyDetailRequest(() => fetchSingleHourlyForecastForCity(city, resolution)),
);
} catch (error) {
rejectBatchWaiters(waiters, error);
} }
resolveBatchWaiters(waiters, null);
}), }),
); );
} catch (error) { } catch (error) {
await Promise.all( resolveAllBatchWaitersAsNull(cities, queue);
cities.map(async (city) => {
const waiters = queue.waiters.get(city);
try {
resolveBatchWaiters(
waiters,
await runQueuedHourlyDetailRequest(() => fetchSingleHourlyForecastForCity(city, resolution)),
);
} catch (fallbackError) {
rejectBatchWaiters(waiters, fallbackError || error);
} }
}),
);
} }
function resolveAllBatchWaitersAsNull(
cities: string[],
queue: CityDetailBatchQueue,
) {
cities.forEach((city) => {
resolveBatchWaiters(queue.waiters.get(city), null);
});
} }
function fetchCityDetailBatchWithTimeout(cities: string[], resolution: string) { function fetchCityDetailBatchWithTimeout(cities: string[], resolution: string) {
@@ -1133,6 +1150,7 @@ function fetchCityDetailBatchWithTimeout(cities: string[], resolution: string) {
force_refresh: "false", force_refresh: "false",
limit: String(Math.max(cities.length, CITY_DETAIL_BATCH_MAX_CITIES)), limit: String(Math.max(cities.length, CITY_DETAIL_BATCH_MAX_CITIES)),
resolution, resolution,
scope: "chart",
}); });
return fetch(`/api/cities/detail-batch?${params.toString()}`, { return fetch(`/api/cities/detail-batch?${params.toString()}`, {
headers: { Accept: "application/json" }, headers: { Accept: "application/json" },
@@ -1158,6 +1176,13 @@ async function fetchHourlyForecastForCity(
if (cached) { if (cached) {
return cached.data; return cached.data;
} }
} else {
const recentlyRefreshed = readHourlyCacheEntry(cacheKey, {
maxAgeMs: HOURLY_FORCE_REFRESH_DEDUP_MS,
});
if (recentlyRefreshed) {
return recentlyRefreshed.data;
}
} }
const requestKey = options.ignoreCache ? `${city}:${resParam}:live` : `${city}:${resParam}`; const requestKey = options.ignoreCache ? `${city}:${resParam}:live` : `${city}:${resParam}`;
@@ -1173,17 +1198,6 @@ async function fetchHourlyForecastForCity(
return request; return request;
} }
function fetchCityDetailWithTimeout(city: string, resolution: string) {
const controller = new AbortController();
const timeoutId = globalThis.setTimeout(() => controller.abort(), HOURLY_DETAIL_REQUEST_TIMEOUT_MS);
return fetch(`/api/city/${encodeURIComponent(city)}/detail?depth=full&force_refresh=false&resolution=${resolution}`, {
headers: { Accept: "application/json" },
signal: controller.signal,
})
.catch(() => null)
.finally(() => globalThis.clearTimeout(timeoutId));
}
function shouldPollLiveChart({ function shouldPollLiveChart({
city, city,
compact, compact,
@@ -2424,8 +2438,10 @@ export {
MAX_HOURLY_DETAIL_CONCURRENT_REQUESTS, MAX_HOURLY_DETAIL_CONCURRENT_REQUESTS,
HOURLY_DETAIL_REQUEST_TIMEOUT_MS, HOURLY_DETAIL_REQUEST_TIMEOUT_MS,
HOURLY_CACHE_TTL_MS, HOURLY_CACHE_TTL_MS,
HOURLY_FORCE_REFRESH_DEDUP_MS,
_hourlyCache, _hourlyCache,
__readHourlyCacheEntryForTest, __readHourlyCacheEntryForTest,
resolveCityDetailFromBatch as __resolveCityDetailFromBatchForTest,
__resetHourlyDetailRequestQueueForTest, __resetHourlyDetailRequestQueueForTest,
__runQueuedHourlyDetailRequestForTest, __runQueuedHourlyDetailRequestForTest,
buildChartDomain, buildChartDomain,
@@ -17,6 +17,7 @@ type LoadTerminalAuthProfileOptions = {
accessToken?: string | null, accessToken?: string | null,
options?: { preferSnapshot?: boolean }, options?: { preferSnapshot?: boolean },
) => Promise<TerminalAuthProfilePayload>; ) => Promise<TerminalAuthProfilePayload>;
timeoutMs?: number;
}; };
type SettledProfile = type SettledProfile =
@@ -57,15 +58,44 @@ function canResolveProfileImmediately(
return payload?.authenticated === true && payload.subscription_active === true; return payload?.authenticated === true && payload.subscription_active === true;
} }
function authProfileRequestCacheKey(
accessToken?: string | null,
options?: { preferSnapshot?: boolean },
) {
const token = String(accessToken || "").trim();
const scope = token ? `bearer:${token}` : "cookie";
const mode = options?.preferSnapshot ? "snapshot" : "live";
return `${mode}:${scope}`;
}
export function createAuthProfileRequestCache(
loadAuthProfile: LoadTerminalAuthProfileOptions["loadAuthProfile"],
): LoadTerminalAuthProfileOptions["loadAuthProfile"] {
const pending = new Map<string, Promise<TerminalAuthProfilePayload>>();
return (accessToken, options) => {
const key = authProfileRequestCacheKey(accessToken, options);
const existing = pending.get(key);
if (existing) return existing;
const request = loadAuthProfile(accessToken, options).finally(() => {
pending.delete(key);
});
pending.set(key, request);
return request;
};
}
export async function loadTerminalAuthProfile({ export async function loadTerminalAuthProfile({
getSession, getSession,
hasSupabasePublicEnv, hasSupabasePublicEnv,
loadAuthProfile, loadAuthProfile,
timeoutMs = 6500,
}: LoadTerminalAuthProfileOptions) { }: LoadTerminalAuthProfileOptions) {
let resolvedAuthenticated = false; let resolvedAuthenticated = false;
let resolveAuthenticated: let resolveAuthenticated:
| ((payload: TerminalAuthProfilePayload) => void) | ((payload: TerminalAuthProfilePayload) => void)
| null = null; | null = null;
let latestCookiePayload: TerminalAuthProfilePayload | null = null;
let latestBearerPayload: TerminalAuthProfilePayload | null = null;
const authenticatedProfile = new Promise<TerminalAuthProfilePayload>((resolve) => { const authenticatedProfile = new Promise<TerminalAuthProfilePayload>((resolve) => {
resolveAuthenticated = resolve; resolveAuthenticated = resolve;
@@ -79,6 +109,7 @@ export async function loadTerminalAuthProfile({
const cookieProfile = settleProfile( const cookieProfile = settleProfile(
loadAuthProfile(null, { preferSnapshot: true }).then((payload) => { loadAuthProfile(null, { preferSnapshot: true }).then((payload) => {
latestCookiePayload = payload;
resolveIfAuthenticated(payload); resolveIfAuthenticated(payload);
return payload; return payload;
}), }),
@@ -94,6 +125,7 @@ export async function loadTerminalAuthProfile({
).trim(); ).trim();
if (!accessToken) return null; if (!accessToken) return null;
const payload = await loadAuthProfile(accessToken, { preferSnapshot: true }); const payload = await loadAuthProfile(accessToken, { preferSnapshot: true });
latestBearerPayload = payload;
resolveIfAuthenticated(payload); resolveIfAuthenticated(payload);
return payload; return payload;
})(), })(),
@@ -103,5 +135,20 @@ export async function loadTerminalAuthProfile({
([cookieResult, bearerResult]) => firstKnownProfile(cookieResult, bearerResult), ([cookieResult, bearerResult]) => firstKnownProfile(cookieResult, bearerResult),
); );
return Promise.race([authenticatedProfile, fallbackProfile]); const timeoutProfile = new Promise<TerminalAuthProfilePayload>((resolve, reject) => {
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return;
globalThis.setTimeout(() => {
if (latestBearerPayload) {
resolve(latestBearerPayload);
return;
}
if (latestCookiePayload) {
resolve(latestCookiePayload);
return;
}
reject(new Error("Terminal auth bootstrap timeout"));
}, timeoutMs);
});
return Promise.race([authenticatedProfile, fallbackProfile, timeoutProfile]);
} }
@@ -0,0 +1,97 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
function readFrontend(...parts: string[]) {
return fs.readFileSync(path.join(process.cwd(), ...parts), "utf8");
}
export function runTests() {
const timingSource = readFrontend("lib", "proxy-timing.ts");
assert.match(
timingSource,
/createProxyTimer/,
"shared proxy timing helper should create timers for slow API proxies",
);
assert.match(
timingSource,
/Server-Timing/,
"shared proxy timing helper should write Server-Timing headers for HAR inspection",
);
assert.doesNotMatch(
timingSource,
/authUserId|authEmail|userId|email/,
"proxy timing logs must avoid raw user ids or emails",
);
const apiProxySource = readFrontend("lib", "api-proxy.ts");
assert.match(
apiProxySource,
/timing\?: ProxyTimer/,
"generic backend JSON proxy should accept an optional timer",
);
for (const stage of ["auth_headers", "backend_fetch", "backend_read"]) {
assert.match(
apiProxySource,
new RegExp(stage),
`generic backend JSON proxy should measure ${stage}`,
);
}
const detailBatchProxy = readFrontend("app", "api", "cities", "detail-batch", "route.ts");
assert.match(detailBatchProxy, /createProxyTimer\(req,\s*"city_detail_batch"\)/);
assert.match(detailBatchProxy, /timing:\s*timer/);
assert.match(
detailBatchProxy,
/fetchCache:\s*"no-store"/,
"city detail batch proxy should avoid caching partial backend fetches in the Next data cache",
);
assert.match(
detailBatchProxy,
/cacheControlForData/,
"city detail batch proxy should be able to suppress response caching for partial payloads",
);
assert.match(
apiProxySource,
/cacheControlForData\?:/,
"generic backend JSON proxy should allow response cache policy to depend on parsed data",
);
const scanTerminalProxy = readFrontend("app", "api", "scan", "terminal", "route.ts");
assert.match(scanTerminalProxy, /createProxyTimer\(req,\s*"scan_terminal"\)/);
assert.match(scanTerminalProxy, /timing:\s*timer/);
const cityDetailProxy = readFrontend("app", "api", "city", "[name]", "detail", "route.ts");
assert.match(cityDetailProxy, /createProxyTimer\(req,\s*"city_detail"\)/);
for (const stage of ["auth_headers", "backend_fetch", "backend_read"]) {
assert.match(cityDetailProxy, new RegExp(stage));
}
const onlineUsersProxy = readFrontend("app", "api", "ops", "online-users", "route.ts");
assert.match(onlineUsersProxy, /createProxyTimer\(req,\s*"ops_online_users"\)/);
for (const stage of ["auth_headers", "ops_auth", "backend_fetch", "backend_read"]) {
assert.match(onlineUsersProxy, new RegExp(stage));
}
const analyticsProxy = readFrontend("app", "api", "analytics", "events", "route.ts");
assert.match(
analyticsProxy,
/ANALYTICS_PROXY_TIMEOUT_MS/,
"analytics event proxy should use a short dedicated timeout instead of waiting for long backend stalls",
);
assert.match(
analyticsProxy,
/createProxyTimer\(req,\s*"analytics_events"\)/,
"analytics event proxy should expose Server-Timing for HAR inspection",
);
assert.match(
analyticsProxy,
/AbortController/,
"analytics event proxy should abort slow upstream tracking requests",
);
assert.match(
analyticsProxy,
/status:\s*202/,
"analytics event proxy timeout should stay non-blocking for the fire-and-forget client event",
);
}
+34 -3
View File
@@ -4,6 +4,10 @@ import { useEffect, useSyncExternalStore } from "react";
import { resolveBackendApiUrl } from "@/lib/backend-api"; import { resolveBackendApiUrl } from "@/lib/backend-api";
const V1_EVENT_TYPE = "city_observation_patch.v1"; const V1_EVENT_TYPE = "city_observation_patch.v1";
const SSE_SUBSCRIPTION_RECONNECT_DELAY_MS = 150;
const SSE_REPLAY_BASE_LIMIT = 60;
const SSE_REPLAY_EVENTS_PER_CITY = 24;
const SSE_REPLAY_MAX_LIMIT = 240;
export type CityPatch = { export type CityPatch = {
type?: string; type?: string;
@@ -38,6 +42,7 @@ const subscribedCities = new Map<string, number>();
let eventSource: EventSource | null = null; let eventSource: EventSource | null = null;
let reconnectTimer: ReturnType<typeof setTimeout> | null = null; let reconnectTimer: ReturnType<typeof setTimeout> | null = null;
let subscriptionReconnectTimer: ReturnType<typeof setTimeout> | null = null;
let reconnectAttempt = 0; let reconnectAttempt = 0;
let patchVersion = 0; let patchVersion = 0;
let resyncVersion = 0; let resyncVersion = 0;
@@ -74,6 +79,12 @@ function clearReconnectTimer() {
reconnectTimer = null; reconnectTimer = null;
} }
function clearSubscriptionReconnectTimer() {
if (!subscriptionReconnectTimer) return;
clearTimeout(subscriptionReconnectTimer);
subscriptionReconnectTimer = null;
}
function buildSseUrl(baseUrl: string) { function buildSseUrl(baseUrl: string) {
const params = new URLSearchParams(); const params = new URLSearchParams();
const cities = subscribedCityList(); const cities = subscribedCityList();
@@ -83,12 +94,17 @@ function buildSseUrl(baseUrl: string) {
if (lastRevision > 0) { if (lastRevision > 0) {
params.set("since_revision", String(lastRevision)); params.set("since_revision", String(lastRevision));
} }
params.set("replay_limit", "500"); params.set("replay_limit", String(resolveSseReplayLimit(cities.length)));
const query = params.toString(); const query = params.toString();
return query ? `${baseUrl}?${query}` : baseUrl; return query ? `${baseUrl}?${query}` : baseUrl;
} }
function resolveSseReplayLimit(cityCount: number) {
const requested = Math.max(1, cityCount) * SSE_REPLAY_EVENTS_PER_CITY;
return Math.max(SSE_REPLAY_BASE_LIMIT, Math.min(SSE_REPLAY_MAX_LIMIT, requested));
}
function currentConnectionKey() { function currentConnectionKey() {
return `${useFallbackUrl ? "fallback" : "direct"}:${subscribedCityList().join("|")}:${lastRevision}`; return `${useFallbackUrl ? "fallback" : "direct"}:${subscribedCityList().join("|")}:${lastRevision}`;
} }
@@ -113,6 +129,7 @@ function closeEventSource() {
function reconnectNow() { function reconnectNow() {
if (typeof window === "undefined") return; if (typeof window === "undefined") return;
clearReconnectTimer(); clearReconnectTimer();
clearSubscriptionReconnectTimer();
closeEventSource(); closeEventSource();
connectSsePatches(); connectSsePatches();
} }
@@ -159,6 +176,7 @@ function connectSsePatches() {
function ensureSsePatchConnection() { function ensureSsePatchConnection() {
if (subscribedCities.size === 0) { if (subscribedCities.size === 0) {
clearSubscriptionReconnectTimer();
closeEventSource(); closeEventSource();
clearReconnectTimer(); clearReconnectTimer();
return; return;
@@ -167,6 +185,19 @@ function ensureSsePatchConnection() {
reconnectNow(); reconnectNow();
} }
function scheduleSubscriptionReconnect() {
if (typeof window === "undefined") return;
if (subscribedCities.size === 0) {
ensureSsePatchConnection();
return;
}
clearSubscriptionReconnectTimer();
subscriptionReconnectTimer = setTimeout(() => {
subscriptionReconnectTimer = null;
ensureSsePatchConnection();
}, SSE_SUBSCRIPTION_RECONNECT_DELAY_MS);
}
function registerCitySubscription(city: string) { function registerCitySubscription(city: string) {
const cityKey = normalizeCityKey(city); const cityKey = normalizeCityKey(city);
if (!cityKey) return () => {}; if (!cityKey) return () => {};
@@ -174,7 +205,7 @@ function registerCitySubscription(city: string) {
const previousCount = subscribedCities.get(cityKey) ?? 0; const previousCount = subscribedCities.get(cityKey) ?? 0;
subscribedCities.set(cityKey, previousCount + 1); subscribedCities.set(cityKey, previousCount + 1);
if (previousCount === 0) { if (previousCount === 0) {
ensureSsePatchConnection(); scheduleSubscriptionReconnect();
} }
return () => { return () => {
@@ -184,7 +215,7 @@ function registerCitySubscription(city: string) {
} else { } else {
subscribedCities.set(cityKey, nextCount); subscribedCities.set(cityKey, nextCount);
} }
ensureSsePatchConnection(); scheduleSubscriptionReconnect();
}; };
} }
+56 -13
View File
@@ -5,6 +5,10 @@ import {
buildBackendRequestHeaders, buildBackendRequestHeaders,
} from "@/lib/backend-auth"; } from "@/lib/backend-auth";
import { buildCachedJsonResponse } from "@/lib/http-cache"; import { buildCachedJsonResponse } from "@/lib/http-cache";
import {
finishProxyTimedResponse,
type ProxyTimer,
} from "@/lib/proxy-timing";
const PASSTHROUGH_UPSTREAM_STATUSES = new Set([ const PASSTHROUGH_UPSTREAM_STATUSES = new Set([
400, 400,
@@ -81,6 +85,7 @@ export async function proxyBackendJsonGet(
req: NextRequest, req: NextRequest,
options: { options: {
cacheControl?: string; cacheControl?: string;
cacheControlForData?: (data: unknown) => string | undefined;
conditionalResponse?: boolean; conditionalResponse?: boolean;
detailLimit?: number; detailLimit?: number;
error?: string; error?: string;
@@ -91,40 +96,75 @@ export async function proxyBackendJsonGet(
signal?: AbortSignal; signal?: AbortSignal;
statusOnException?: number; statusOnException?: number;
timeoutPublicMessage?: string; timeoutPublicMessage?: string;
timing?: ProxyTimer;
url: string; url: string;
}, },
) { ) {
let auth: Awaited<ReturnType<typeof buildBackendRequestHeaders>> | null = null; let auth: Awaited<ReturnType<typeof buildBackendRequestHeaders>> | null = null;
const timing = options.timing;
try { try {
auth = await buildBackendRequestHeaders(req, { auth = await (timing
? timing.measure("auth_headers", () =>
buildBackendRequestHeaders(req, {
includeSupabaseIdentity: options.includeSupabaseIdentity ?? false, includeSupabaseIdentity: options.includeSupabaseIdentity ?? false,
}); }),
const res = await fetch(options.url, { )
: buildBackendRequestHeaders(req, {
includeSupabaseIdentity: options.includeSupabaseIdentity ?? false,
}));
const res = await (timing
? timing.measure("backend_fetch", () =>
fetch(options.url, {
headers: auth!.headers,
...(options.fetchCache
? { cache: options.fetchCache }
: { next: { revalidate: options.revalidateSeconds ?? 30 } }),
signal: options.signal,
}),
)
: fetch(options.url, {
headers: auth.headers, headers: auth.headers,
...(options.fetchCache ...(options.fetchCache
? { cache: options.fetchCache } ? { cache: options.fetchCache }
: { next: { revalidate: options.revalidateSeconds ?? 30 } }), : { next: { revalidate: options.revalidateSeconds ?? 30 } }),
signal: options.signal, signal: options.signal,
}); }));
const backendServerTiming = res.headers.get("server-timing") || "";
if (!res.ok) { if (!res.ok) {
const raw = await res.text(); const raw = await (timing
? timing.measure("backend_read", () => res.text())
: res.text());
const response = buildUpstreamErrorResponse(res.status, raw, { const response = buildUpstreamErrorResponse(res.status, raw, {
detailLimit: options.detailLimit, detailLimit: options.detailLimit,
error: options.error, error: options.error,
}); });
return applyAuthResponseCookies(response, auth.response); const withCookies = applyAuthResponseCookies(response, auth.response);
return timing
? finishProxyTimedResponse(withCookies, timing, `upstream_${res.status}`, {
backendServerTiming,
})
: withCookies;
} }
const data = await res.json(); const data = await (timing
? timing.measure("backend_read", () => res.json())
: res.json());
const responseCacheControl =
options.cacheControlForData?.(data) ?? options.cacheControl;
const response = const response =
options.cacheControl && options.conditionalResponse !== false responseCacheControl && options.conditionalResponse !== false
? buildCachedJsonResponse(req, data, options.cacheControl) ? buildCachedJsonResponse(req, data, responseCacheControl)
: NextResponse.json(data, { : NextResponse.json(data, {
headers: options.cacheControl headers: responseCacheControl
? { "Cache-Control": options.cacheControl } ? { "Cache-Control": responseCacheControl }
: undefined, : undefined,
}); });
return applyAuthResponseCookies(response, auth.response); const withCookies = applyAuthResponseCookies(response, auth.response);
return timing
? finishProxyTimedResponse(withCookies, timing, "ok", {
backendServerTiming,
})
: withCookies;
} catch (error) { } catch (error) {
const timedOut = options.signal?.aborted === true; const timedOut = options.signal?.aborted === true;
const response = buildProxyExceptionResponse(error, { const response = buildProxyExceptionResponse(error, {
@@ -134,6 +174,9 @@ export async function proxyBackendJsonGet(
: options.publicMessage, : options.publicMessage,
status: timedOut ? 504 : options.statusOnException, status: timedOut ? 504 : options.statusOnException,
}); });
return auth ? applyAuthResponseCookies(response, auth.response) : response; const withCookies = auth ? applyAuthResponseCookies(response, auth.response) : response;
return timing
? finishProxyTimedResponse(withCookies, timing, timedOut ? "timeout" : "exception")
: withCookies;
} }
} }
+1
View File
@@ -716,6 +716,7 @@ export interface AiAnalysisStructured {
} }
export interface CityDetail { export interface CityDetail {
city?: string;
name: string; name: string;
display_name: string; display_name: string;
detail_depth?: "panel" | "market" | "nearby" | "full"; detail_depth?: "panel" | "market" | "nearby" | "full";
+19 -1
View File
@@ -28,4 +28,22 @@ export function buildForceRefreshProxyCachePolicy(
}; };
} }
export const buildCityDetailProxyCachePolicy = buildForceRefreshProxyCachePolicy; export function buildCityDetailProxyCachePolicy(
forceRefresh: string | null | undefined,
revalidateSeconds = 15,
): ProxyCachePolicy {
if (isForceRefreshValue(forceRefresh)) {
return {
fetchMode: "no-store",
responseCacheControl: "no-store, max-age=0",
};
}
return {
fetchMode: "revalidate",
responseCacheControl: `public, max-age=${revalidateSeconds}, s-maxage=${revalidateSeconds}, stale-while-revalidate=${Math.max(
revalidateSeconds * 3,
30,
)}`,
revalidateSeconds,
};
}
+113
View File
@@ -0,0 +1,113 @@
import type { NextRequest, NextResponse } from "next/server";
export type ProxyTimingStage = {
durationMs: number;
name: string;
};
export type ProxyTimer = {
hasAuthorization: boolean;
hasSupabaseCookie: boolean;
measure<T>(name: string, action: () => Promise<T>): Promise<T>;
measureSync<T>(name: string, action: () => T): T;
route: string;
stages: ProxyTimingStage[];
totalMs(): number;
};
function proxyNowMs() {
return typeof performance !== "undefined" ? performance.now() : Date.now();
}
function hasRequestSupabaseSessionCookie(req: NextRequest) {
return req.cookies.getAll().some((cookie) => {
const name = cookie.name.toLowerCase();
const value = String(cookie.value || "").trim();
return Boolean(
value &&
(name === "supabase-auth-token" ||
(name.startsWith("sb-") && name.includes("-auth-token"))),
);
});
}
export function createProxyTimer(req: NextRequest, route: string): ProxyTimer {
const startedAt = proxyNowMs();
const stages: ProxyTimingStage[] = [];
const recordStage = (name: string, stageStartedAt: number) => {
stages.push({
durationMs: Math.round((proxyNowMs() - stageStartedAt) * 10) / 10,
name,
});
};
return {
hasAuthorization: Boolean(req.headers.get("authorization")),
hasSupabaseCookie: hasRequestSupabaseSessionCookie(req),
async measure<T>(name: string, action: () => Promise<T>) {
const stageStartedAt = proxyNowMs();
try {
return await action();
} finally {
recordStage(name, stageStartedAt);
}
},
measureSync<T>(name: string, action: () => T) {
const stageStartedAt = proxyNowMs();
try {
return action();
} finally {
recordStage(name, stageStartedAt);
}
},
route,
stages,
totalMs() {
return Math.round((proxyNowMs() - startedAt) * 10) / 10;
},
};
}
function formatServerTiming(stages: ProxyTimingStage[]) {
return stages
.map(({ durationMs, name }) => {
const safeName = name.replace(/[^A-Za-z0-9_-]/g, "_");
return `${safeName};dur=${Math.max(0, durationMs).toFixed(1)}`;
})
.join(", ");
}
export function finishProxyTimedResponse(
response: NextResponse,
timer: ProxyTimer,
outcome: string,
extra?: { backendServerTiming?: string },
) {
const total = timer.totalMs();
const ownServerTiming = formatServerTiming(
[...timer.stages, { durationMs: total, name: "total" }].map((stage) => ({
...stage,
name: `${timer.route}_${stage.name}`,
})),
);
const backendServerTiming = String(extra?.backendServerTiming || "").trim();
response.headers.set(
"Server-Timing",
backendServerTiming
? `${ownServerTiming}, ${backendServerTiming}`
: ownServerTiming,
);
console.info(
"[api-proxy-timing]",
JSON.stringify({
hasAuthorization: timer.hasAuthorization,
hasSupabaseCookie: timer.hasSupabaseCookie,
outcome,
route: timer.route,
stages: timer.stages,
status: response.status,
totalMs: total,
}),
);
return response;
}
+120
View File
@@ -0,0 +1,120 @@
from pathlib import Path
from fastapi.testclient import TestClient
from web.app import app
import web.services.city_api as city_api
import web.services.scan_api as scan_api
ROOT = Path(__file__).resolve().parents[1]
client = TestClient(app)
def test_backend_shared_timing_helper_avoids_sensitive_identity_fields():
source = (ROOT / "web" / "services" / "request_timing.py").read_text(
encoding="utf-8"
)
assert "ServerTimingRecorder" in source
assert "server_timing_value" in source
assert "user_id" not in source
assert "email" not in source
def test_city_detail_batch_response_includes_backend_server_timing(monkeypatch):
city_api._CITY_DETAIL_BATCH_RESPONSE_CACHE.clear()
city_api._CITY_DETAIL_BATCH_RESPONSE_CACHE_TS.clear()
city_api._CITY_DETAIL_BATCH_RESPONSE_INFLIGHT.clear()
class FakeCache:
def get_city_cache(self, kind, city):
assert kind == "full"
return {"payload": {"city": city, "hourly": {"times": [], "temps": []}}}
def build_detail(data, market_slug, target_date, resolution):
return {"city": data["city"], "resolution": resolution}
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(
city_api.legacy_routes,
"_normalize_city_or_404",
lambda name: name.strip().lower(),
)
monkeypatch.setattr(city_api.legacy_routes, "_city_cache_is_fresh", lambda entry, ttl: True)
monkeypatch.setattr(
city_api.legacy_routes,
"_overlay_latest_wunderground_current",
lambda city, payload: payload,
)
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeCache())
monkeypatch.setattr(city_api.legacy_routes, "_build_city_detail_payload", build_detail)
response = client.get("/api/cities/detail-batch?cities=Paris&resolution=10m")
assert response.status_code == 200
server_timing = response.headers["server-timing"]
assert "city_detail_batch_assert_entitlement" in server_timing
assert "city_detail_batch_full_data_paris" in server_timing
assert "city_detail_batch_detail_payload_paris" in server_timing
assert "city_detail_batch_total" in server_timing
def test_city_detail_response_includes_backend_server_timing(monkeypatch):
class FakeCache:
def get_city_cache(self, kind, city):
assert kind == "full"
return {"payload": {"city": city, "hourly": {"times": [], "temps": []}}}
def build_detail(data, market_slug, target_date, resolution):
return {"city": data["city"], "resolution": resolution}
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(
city_api.legacy_routes,
"_normalize_city_or_404",
lambda name: name.strip().lower(),
)
monkeypatch.setattr(city_api.legacy_routes, "_city_cache_is_fresh", lambda entry, ttl: True)
monkeypatch.setattr(
city_api.legacy_routes,
"_overlay_latest_wunderground_current",
lambda city, payload: payload,
)
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeCache())
monkeypatch.setattr(city_api.legacy_routes, "_build_city_detail_payload", build_detail)
response = client.get("/api/city/Paris/detail?resolution=10m")
assert response.status_code == 200
server_timing = response.headers["server-timing"]
assert "city_detail_assert_entitlement" in server_timing
assert "city_detail_full_data" in server_timing
assert "city_detail_detail_payload" in server_timing
assert "city_detail_total" in server_timing
def test_scan_terminal_response_includes_backend_server_timing(monkeypatch):
monkeypatch.setattr(scan_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(
scan_api.legacy_routes,
"build_scan_terminal_payload",
lambda filters, force_refresh=False, timing_recorder=None: {"rows": [], "filters": filters},
)
response = client.get("/api/scan/terminal?limit=1")
assert response.status_code == 200
server_timing = response.headers["server-timing"]
assert "scan_terminal_assert_entitlement" in server_timing
assert "scan_terminal_build_payload" in server_timing
assert "scan_terminal_total" in server_timing
def test_online_users_response_includes_backend_server_timing():
response = client.get("/api/ops/online-users")
assert response.status_code == 200
server_timing = response.headers["server-timing"]
assert "ops_online_users_online_count" in server_timing
assert "ops_online_users_total" in server_timing
+44 -1
View File
@@ -38,15 +38,49 @@ def test_scan_terminal_prewarm_is_lazy_by_default():
) )
def test_scan_terminal_prewarm_only_runs_for_web_service(monkeypatch):
from web import app_factory
monkeypatch.setenv("POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED", "true")
monkeypatch.delenv("POLYWEATHER_SERVICE_ROLE", raising=False)
assert app_factory._scan_terminal_prewarm_enabled() is False
monkeypatch.setenv("POLYWEATHER_SERVICE_ROLE", "bot")
assert app_factory._scan_terminal_prewarm_enabled() is False
monkeypatch.setenv("POLYWEATHER_SERVICE_ROLE", "web")
assert app_factory._scan_terminal_prewarm_enabled() is True
def test_docker_compose_isolates_scan_terminal_prewarm_to_web_service():
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "POLYWEATHER_SERVICE_ROLE: web" in compose
assert "POLYWEATHER_SERVICE_ROLE: bot" in compose
assert "POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED: 'false'" in compose
def test_scan_terminal_backend_timeout_returns_before_next_proxy_abort(): def test_scan_terminal_backend_timeout_returns_before_next_proxy_abort():
import web.services.scan_terminal_config as scan_terminal_config import web.services.scan_terminal_config as scan_terminal_config
route_source = ( route_source = (
ROOT / "frontend" / "app" / "api" / "scan" / "terminal" / "route.ts" ROOT / "frontend" / "app" / "api" / "scan" / "terminal" / "route.ts"
).read_text(encoding="utf-8") ).read_text(encoding="utf-8")
config_source = (
ROOT / "web" / "services" / "scan_terminal_config.py"
).read_text(encoding="utf-8")
assert 'POLYWEATHER_SCAN_TERMINAL_PROXY_TIMEOUT_MS || "40000"' in route_source assert 'POLYWEATHER_SCAN_TERMINAL_PROXY_TIMEOUT_MS || "18000"' in route_source
assert '"POLYWEATHER_SCAN_TERMINAL_BUILD_TIMEOUT_SEC",\n 10,' in config_source
assert (
'"POLYWEATHER_SCAN_TERMINAL_PREWARM_PAYLOAD_TIMEOUT_SEC",\n 30,'
in config_source
)
assert scan_terminal_config.SCAN_TERMINAL_BUILD_TIMEOUT_SEC <= 30 assert scan_terminal_config.SCAN_TERMINAL_BUILD_TIMEOUT_SEC <= 30
assert (
scan_terminal_config.SCAN_TERMINAL_PREWARM_PAYLOAD_TIMEOUT_SEC
>= scan_terminal_config.SCAN_TERMINAL_BUILD_TIMEOUT_SEC
)
def test_probability_engine_uses_enriched_multi_model_snapshot(): def test_probability_engine_uses_enriched_multi_model_snapshot():
@@ -80,6 +114,15 @@ def test_deploy_script_retries_startup_smoke_checks():
assert 'smoke_check "frontend" "https://www.polyweather.top/" 15 3 5' in script assert 'smoke_check "frontend" "https://www.polyweather.top/" 15 3 5' in script
def test_docker_compose_keeps_polyweather_ports_on_loopback():
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
assert "127.0.0.1:3001:3000" in compose
assert "127.0.0.1:8000:8000" in compose
assert "\n - 3001:3000" not in compose
assert "\n - 8000:8000" not in compose
def test_city_detail_builds_deb_hourly_consensus_before_peak_window(): def test_city_detail_builds_deb_hourly_consensus_before_peak_window():
source = (ROOT / "web" / "analysis_service.py").read_text(encoding="utf-8") source = (ROOT / "web" / "analysis_service.py").read_text(encoding="utf-8")
+144 -1
View File
@@ -1,12 +1,84 @@
from web.scan_terminal_filters import normalize_scan_terminal_filters from web.scan_terminal_filters import normalize_scan_terminal_filters
from web import scan_terminal_cache
from web.scan_terminal_metar_gate import _apply_metar_gate_to_row from web.scan_terminal_metar_gate import _apply_metar_gate_to_row
from web.scan_terminal_payloads import ( from web.scan_terminal_payloads import (
build_failed_scan_terminal_payload, build_failed_scan_terminal_payload,
build_scan_terminal_snapshot_id, build_scan_terminal_snapshot_id,
build_stale_scan_terminal_payload, build_stale_scan_terminal_payload,
compact_ranked_scan_rows_for_payload,
SCAN_PAYLOAD_DEFERRED_RUNWAY_POINTS,
SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS,
) )
from web.scan_terminal_ranker import build_ranked_scan_terminal_result from web.scan_terminal_ranker import build_ranked_scan_terminal_result
from web.scan_terminal_city_row import _build_quick_row
from web.routers.scan import router as scan_router from web.routers.scan import router as scan_router
from web.scan_terminal_service import _scan_terminal_prewarm_filters
class _FakeRedis:
def __init__(self):
self.data = {}
def get(self, key):
return self.data.get(key)
def setex(self, key, _ttl, value):
self.data[key] = value
def test_scan_terminal_cache_hydrates_success_payload_from_redis(monkeypatch):
fake_redis = _FakeRedis()
monkeypatch.setenv("POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_ENABLED", "true")
monkeypatch.setattr(scan_terminal_cache, "_get_redis_client", lambda: fake_redis)
scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
filters = {"scan_mode": "tradable", "limit": 9}
payload = {
"generated_at": "2026-06-01T00:00:00Z",
"rows": [{"id": "row-1"}],
"summary": {"candidate_total": 1},
}
scan_terminal_cache.set_cached_scan_terminal_payload(filters, payload)
scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
entry = scan_terminal_cache.get_scan_terminal_cache_entry(filters)
cached = scan_terminal_cache.get_cached_scan_terminal_payload(filters, ttl_sec=3600)
assert entry["success_payload"]["rows"] == [{"id": "row-1"}]
assert cached["summary"]["candidate_total"] == 1
def test_scan_terminal_failure_state_preserves_redis_success_payload(monkeypatch):
fake_redis = _FakeRedis()
monkeypatch.setenv("POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_ENABLED", "true")
monkeypatch.setattr(scan_terminal_cache, "_get_redis_client", lambda: fake_redis)
scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
filters = {"scan_mode": "tradable", "limit": 9}
scan_terminal_cache.set_cached_scan_terminal_payload(
filters,
{
"generated_at": "2026-06-01T00:00:00Z",
"rows": [{"id": "row-1"}],
},
)
scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
scan_terminal_cache.set_scan_terminal_failure_state(filters, error_message="timeout")
scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
entry = scan_terminal_cache.get_scan_terminal_cache_entry(filters)
assert entry["success_payload"]["rows"] == [{"id": "row-1"}]
assert entry["last_error"] == "timeout"
def test_scan_terminal_prewarm_covers_default_api_limit():
limits = {filters["limit"] for filters in _scan_terminal_prewarm_filters()}
assert 25 in limits
assert 180 in limits
def test_scan_router_does_not_expose_terminal_ai_endpoint(): def test_scan_router_does_not_expose_terminal_ai_endpoint():
@@ -137,6 +209,78 @@ def test_scan_terminal_snapshot_id_is_stable_for_same_ranked_inputs():
assert first.startswith("scan-") assert first.startswith("scan-")
def test_scan_terminal_payload_slims_deferred_runway_history_rows():
runway_points = [
{"time": f"2026-05-31T{hour:02d}:00:00+00:00", "temp": 20 + hour}
for hour in range(24)
]
rows = [
{
"id": f"row-{index}",
"runway_plate_history": {"35R": list(runway_points)},
}
for index in range(SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS + 2)
]
compacted = compact_ranked_scan_rows_for_payload(rows)
assert len(compacted[0]["runway_plate_history"]["35R"]) == len(runway_points)
assert (
len(
compacted[SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS - 1][
"runway_plate_history"
]["35R"]
)
== len(runway_points)
)
assert (
len(
compacted[SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS][
"runway_plate_history"
]["35R"]
)
== SCAN_PAYLOAD_DEFERRED_RUNWAY_POINTS
)
assert len(rows[SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS]["runway_plate_history"]["35R"]) == len(
runway_points
)
def test_scan_terminal_quick_row_compacts_runway_history_for_list_payload():
raw_history = {
"35R": [
{"time": "2026-05-31T00:00:00+00:00", "temp": 22.11},
{"time": "2026-05-31T00:01:00+00:00", "temp": 22.22},
{"time": "2026-05-31T00:02:00+00:00", "temp": 22.33},
{"time": "2026-05-31T00:10:00+00:00", "temp": 23.44},
{"time": "2026-05-31T00:11:00+00:00", "temp": 23.55},
]
}
row = _build_quick_row(
city="shanghai",
data={
"display_name": "Shanghai",
"local_date": "2026-05-31",
"local_time": "2026-05-31T08:11:00+08:00",
"temp_symbol": "°C",
"current": {"temp": 22.3, "max_so_far": 23.0},
"risk": {"airport": "Shanghai Pudong", "level": "medium"},
"deb": {"prediction": 24.0},
"probabilities": {"distribution": []},
"multi_model": {},
"runway_plate_history": raw_history,
},
)
compact_history = row["runway_plate_history"]["35R"]
assert len(compact_history) == 2
assert compact_history[0]["temp"] == 22.3
assert compact_history[1]["temp"] == 23.6
assert len(str(row["runway_plate_history"])) < len(str(raw_history))
def test_metar_gate_vetoes_yes_when_observed_breaks_above_bucket(): def test_metar_gate_vetoes_yes_when_observed_breaks_above_bucket():
row = { row = {
"id": "yes-row", "id": "yes-row",
@@ -158,4 +302,3 @@ def test_metar_gate_vetoes_yes_when_observed_breaks_above_bucket():
assert row["v4_metar_decision"] == "veto" assert row["v4_metar_decision"] == "veto"
assert row["ai_decision"] == "veto" assert row["ai_decision"] == "veto"
assert "越过目标桶上沿" in row["ai_reason_zh"] assert "越过目标桶上沿" in row["ai_reason_zh"]
+119 -2
View File
@@ -129,8 +129,125 @@ def test_events_endpoint_emits_resync_when_replay_is_incomplete(monkeypatch):
def test_replay_limit_is_bounded(): def test_replay_limit_is_bounded():
assert sse_router._bounded_replay_limit(0) == 1 assert sse_router._bounded_replay_limit(0) == 1
assert sse_router._bounded_replay_limit(500) == 500 assert sse_router._bounded_replay_limit(500) == 60
assert sse_router._bounded_replay_limit(5000) == 2000 assert sse_router._bounded_replay_limit(5000) == 60
assert sse_router._bounded_replay_limit(500, city_count=5) == 120
assert sse_router._bounded_replay_limit(500, city_count=20) == 240
assert sse_router._bounded_replay_limit(25, city_count=5) == 25
def test_replay_gap_direct_resync_policy():
assert (
sse_router._should_direct_resync(
since_revision=1000,
latest_revision=1200,
limit=60,
)
is False
)
assert (
sse_router._should_direct_resync(
since_revision=1000,
latest_revision=1300,
limit=60,
)
is True
)
assert (
sse_router._should_direct_resync(
since_revision=0,
latest_revision=1300,
limit=60,
)
is False
)
def test_legacy_high_replay_limit_is_clamped_by_city_count(monkeypatch):
captured = {}
class FakeStore:
def latest_revision(self):
return 44
def replay_events(self, *, cities, since_revision, limit):
captured["cities"] = cities
captured["limit"] = limit
return []
def replay_requires_resync(self, *, cities, since_revision, replay_count, limit):
captured["resync_limit"] = limit
return False
async def finite_stream(
user_id,
*,
cities=None,
replay_events=None,
connected_revision=0,
resync_event=None,
):
yield sse_router.sse_manager._format_event(
{"type": "connected", "revision": connected_revision}
)
monkeypatch.setattr(sse_router, "event_store", FakeStore())
monkeypatch.setattr(sse_router.sse_manager, "event_stream", finite_stream)
response = TestClient(app).get(
"/api/events?cities=ankara,buenos%20aires,istanbul,jeddah,seoul"
"&since_revision=42&replay_limit=500"
)
assert response.status_code == 200
assert captured["cities"] == {"ankara", "buenos aires", "istanbul", "jeddah", "seoul"}
assert captured["limit"] == 120
assert captured["resync_limit"] == 120
def test_stale_client_gets_direct_resync_without_replay_scan(monkeypatch):
calls = {"replay": 0, "requires_resync": 0}
class FakeStore:
def latest_revision(self):
return 2000
def replay_events(self, *, cities, since_revision, limit):
calls["replay"] += 1
return []
def replay_requires_resync(self, *, cities, since_revision, replay_count, limit):
calls["requires_resync"] += 1
return False
async def finite_stream(
user_id,
*,
cities=None,
replay_events=None,
connected_revision=0,
resync_event=None,
):
yield sse_router.sse_manager._format_event(
{"type": "connected", "revision": connected_revision}
)
if resync_event:
yield sse_router.sse_manager._format_event(resync_event)
monkeypatch.setattr(sse_router, "event_store", FakeStore())
monkeypatch.setattr(sse_router.sse_manager, "event_stream", finite_stream)
response = TestClient(app).get(
"/api/events?cities=ankara,buenos%20aires,istanbul,jeddah,seoul"
"&since_revision=100&replay_limit=500"
)
assert response.status_code == 200
assert calls == {"replay": 0, "requires_resync": 0}
events = _decode_sse_events(response.text)
assert events[-1]["type"] == "resync_required"
assert events[-1]["reason"] == "replay_gap_too_large"
assert events[-1]["latest_revision"] == 2000
def test_ingest_patch_uses_external_fanout_without_direct_broadcast(monkeypatch): def test_ingest_patch_uses_external_fanout_without_direct_broadcast(monkeypatch):
+546
View File
@@ -31,6 +31,27 @@ def test_healthz_returns_ok_shape():
assert 'cities_count' in payload assert 'cities_count' in payload
def test_healthz_keeps_liveness_200_when_db_health_is_degraded(monkeypatch):
from web.services import system_api
monkeypatch.setattr(
system_api,
"build_health_payload",
lambda: {
"status": "degraded",
"time_utc": "2026-05-30T00:00:00+00:00",
"db": {"ok": False, "error": "database is locked"},
"state_storage_mode": "sqlite",
"cities_count": 50,
},
)
response = client.get("/healthz")
assert response.status_code == 200
assert response.json()["status"] == "degraded"
def test_system_status_returns_summary_shape(): def test_system_status_returns_summary_shape():
response = client.get('/api/system/status') response = client.get('/api/system/status')
assert response.status_code == 200 assert response.status_code == 200
@@ -315,6 +336,15 @@ def test_ops_billing_risk_surfaces_trial_payment_referral_and_points(monkeypatch
DBManager, DBManager,
"list_app_analytics_events", "list_app_analytics_events",
lambda self, limit=20000, since_iso=None: [ lambda self, limit=20000, since_iso=None: [
{
"id": 10,
"event_type": "login_start",
"user_id": None,
"client_id": "c1",
"session_id": "session-gap",
"created_at": recent,
"payload": {"mode": "signup"},
},
{ {
"id": 11, "id": 11,
"event_type": "signup_success", "event_type": "signup_success",
@@ -419,6 +449,108 @@ def test_ops_billing_risk_does_not_flag_signup_when_backend_trial_exists(monkeyp
assert not any(issue["category"] == "signup_trial" for issue in payload["issues"]) assert not any(issue["category"] == "signup_trial" for issue in payload["issues"])
def test_ops_billing_risk_ignores_account_visit_without_signup_intent(monkeypatch):
from src.database.db_manager import DBManager
recent = datetime.now(timezone.utc).isoformat()
monkeypatch.setattr(ops_api.legacy_routes, "_require_ops_admin", lambda request: {"email": "ops@example.com"})
monkeypatch.setattr(ops_api, "_supabase_rest_rows", lambda table, params, *, timeout=10: [])
monkeypatch.setattr(
DBManager,
"list_app_analytics_events",
lambda self, limit=20000, since_iso=None: [
{
"id": 61,
"event_type": "signup_success",
"user_id": "returning-no-trial-user",
"client_id": "client-return",
"session_id": "session-return",
"created_at": recent,
"payload": {
"entry": "account_center",
"user_id": "returning-no-trial-user",
},
}
],
)
monkeypatch.setattr(
DBManager,
"list_payment_audit_events",
lambda self, limit=50, event_type=None: [],
)
payload = ops_api.get_ops_billing_risk(None, days=30, limit=20)
assert payload["summary"]["trial_gaps"] == 0
assert not any(issue["category"] == "signup_trial" for issue in payload["issues"])
def test_ops_billing_risk_treats_expired_signup_trial_subscription_as_backend_evidence(monkeypatch):
from src.database.db_manager import DBManager
now = datetime.now(timezone.utc)
recent = now.isoformat()
expired = (now - timedelta(days=2)).isoformat()
def fake_supabase_rows(table, params, *, timeout=10):
if table == "subscriptions" and params.get("or") == "(source.eq.signup_trial,plan_code.eq.signup_trial_3d)":
return [
{
"id": 81,
"user_id": "expired-trial-user",
"plan_code": "signup_trial_3d",
"source": "signup_trial",
"status": "expired",
"starts_at": (now - timedelta(days=5)).isoformat(),
"expires_at": expired,
"created_at": (now - timedelta(days=5)).isoformat(),
"updated_at": expired,
}
]
return []
monkeypatch.setattr(ops_api.legacy_routes, "_require_ops_admin", lambda request: {"email": "ops@example.com"})
monkeypatch.setattr(ops_api, "_supabase_rest_rows", fake_supabase_rows)
monkeypatch.setattr(
DBManager,
"list_app_analytics_events",
lambda self, limit=20000, since_iso=None: [
{
"id": 80,
"event_type": "login_start",
"user_id": None,
"client_id": "client-expired",
"session_id": "session-expired",
"created_at": recent,
"payload": {"mode": "signup"},
},
{
"id": 82,
"event_type": "signup_success",
"user_id": "expired-trial-user",
"client_id": "client-expired",
"session_id": "session-expired",
"created_at": recent,
"payload": {
"entry": "account_center",
"user_id": "expired-trial-user",
},
},
],
)
monkeypatch.setattr(
DBManager,
"list_payment_audit_events",
lambda self, limit=50, event_type=None: [],
)
payload = ops_api.get_ops_billing_risk(None, days=30, limit=20)
assert payload["summary"]["trial_gaps"] == 0
assert not any(issue["category"] == "signup_trial" for issue in payload["issues"])
def test_ops_payment_incidents_expose_top_level_reason_and_filters_resolved(monkeypatch): def test_ops_payment_incidents_expose_top_level_reason_and_filters_resolved(monkeypatch):
from src.database.db_manager import DBManager from src.database.db_manager import DBManager
@@ -567,6 +699,119 @@ def test_city_detail_batch_endpoint_builds_multiple_cached_details(monkeypatch):
assert sorted(calls) == [("paris", "10m"), ("shanghai", "10m")] assert sorted(calls) == [("paris", "10m"), ("shanghai", "10m")]
def test_city_detail_batch_chart_scope_returns_only_chart_fields(monkeypatch):
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(
city_api.legacy_routes,
"_city_cache_is_fresh",
lambda entry, ttl: True,
)
monkeypatch.setattr(
city_api.legacy_routes,
"_overlay_latest_wunderground_current",
lambda city, payload: payload,
)
class FakeCache:
def get_city_cache(self, kind, city):
assert kind == "full"
return {
"payload": {
"name": city,
"display_name": city.title(),
"local_date": "2026-05-30",
"local_time": "15:20",
"temp_symbol": "°C",
"current": {
"temp": 20.0,
"settlement_source": "metar",
"settlement_source_label": "METAR",
},
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [20.0]},
"forecast": {
"today_high": 22.0,
"daily": [{"date": "2026-05-30", "max_temp": 22.0}],
},
"multi_model": {
"hourly_times": ["15:00"],
"hourly_forecasts": {"ECMWF": [21.0]},
},
"deb": {"prediction": 21.5, "hourly_path": {"times": ["15:00"], "temps": [21.5]}},
"probabilities": {"mu": 21.4, "distribution": [{"value": 21, "probability": 0.4}]},
"runway_plate_history": {"01/19": [{"time": "2026-05-30T15:20:00Z", "temp": 20.1}]},
"airport_current": {"temp": 20.0},
"airport_primary": {"temp": 20.0},
"airport_primary_today_obs": [["15:20", 20.0]],
"wunderground_current": {"max_so_far": 20.5},
"settlement_station": {"settlement_station_label": "Station"},
"amos": {"runway_obs": {"point_temperatures": []}},
"metar_today_obs": [{"time": "15:20", "temp": 20.0}],
"settlement_today_obs": [],
"dynamic_commentary": {"summary": "large text"},
"official_nearby": [{"name": "unused"}],
"taf": {"raw": "unused"},
"ai_analysis": "unused",
}
}
def build_detail(_data, _market_slug, _target_date, _resolution):
raise AssertionError("chart scope must not build the full city detail payload")
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeCache())
monkeypatch.setattr(city_api.legacy_routes, "_build_city_detail_payload", build_detail)
response = client.get("/api/cities/detail-batch?cities=Paris&resolution=10m&scope=chart")
assert response.status_code == 200
detail = response.json()["details"]["paris"]
assert detail["timeseries"]["hourly"]["temps"] == [20.0]
assert detail["models_hourly"]["curves"]["ECMWF"] == [21.0]
assert detail["deb"]["hourly_path"]["temps"] == [21.5]
assert detail["airport_primary_today_obs"] == [["15:20", 20.0]]
assert "dynamic_commentary" not in detail
assert "official_nearby" not in detail
assert "taf" not in detail
assert "ai_analysis" not in detail
def test_chart_detail_payload_avoids_threadpool_queue_and_reuses_short_cache(monkeypatch):
import asyncio
build_calls = 0
async def fail_threadpool(*_args, **_kwargs):
raise AssertionError("chart payload should not wait behind the shared threadpool")
def build_chart_detail(data, resolution):
nonlocal build_calls
build_calls += 1
return {
"city": data["city"],
"resolution": resolution,
"hourly": data["hourly"],
}
city_api._CITY_CHART_DETAIL_PAYLOAD_CACHE.clear()
city_api._CITY_CHART_DETAIL_PAYLOAD_CACHE_TS.clear()
monkeypatch.setenv("POLYWEATHER_CITY_DETAIL_PAYLOAD_CACHE_TTL_SEC", "20")
monkeypatch.setattr(city_api, "run_in_threadpool", fail_threadpool)
monkeypatch.setattr(city_api.legacy_routes, "_build_city_chart_detail_payload", build_chart_detail)
data = {
"city": "paris",
"updated_at": "2026-05-30T15:00:00Z",
"hourly": {"times": ["2026-05-30T15:00:00Z"], "temps": [20.0]},
}
first = asyncio.run(city_api._build_city_chart_detail_payload(data, "10m"))
second = asyncio.run(city_api._build_city_chart_detail_payload(data, "10m"))
assert first == second
assert first["resolution"] == "10m"
assert build_calls == 1
def test_city_detail_batch_endpoint_limits_backend_concurrency(monkeypatch): def test_city_detail_batch_endpoint_limits_backend_concurrency(monkeypatch):
import asyncio import asyncio
@@ -618,6 +863,147 @@ def test_city_detail_batch_endpoint_limits_backend_concurrency(monkeypatch):
assert max_active <= 2 assert max_active <= 2
def test_city_detail_batch_returns_completed_details_when_one_city_is_slow(monkeypatch):
import asyncio
completed = []
async def build_batch_item(city, **kwargs):
if city == "slow":
await asyncio.sleep(0.08)
completed.append(city)
return city, {
"city": city,
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [20.0]},
"resolution": kwargs.get("resolution"),
}
monkeypatch.setenv("POLYWEATHER_CITY_DETAIL_BATCH_PARTIAL_TIMEOUT_MS", "20")
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api, "_build_city_detail_batch_item_async", build_batch_item)
payload = asyncio.run(
city_api.get_city_detail_batch_payload(
object(),
cities="fast,slow,other",
resolution="10m",
limit=3,
)
)
assert payload["cities"] == ["fast", "slow", "other"]
assert sorted(payload["details"]) == ["fast", "other"]
assert payload["details"]["fast"]["resolution"] == "10m"
assert payload["partial"] is True
assert payload["missing"] == ["slow"]
assert payload["errors"] == {}
assert "slow" not in completed
def test_city_detail_batch_response_cache_keeps_entitlement_check(monkeypatch):
import asyncio
entitlement_calls = 0
build_calls = 0
async def build_batch_item(city, **kwargs):
nonlocal build_calls
build_calls += 1
return city, {
"city": city,
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [20.0]},
"resolution": kwargs.get("resolution"),
}
def assert_entitlement(request):
nonlocal entitlement_calls
entitlement_calls += 1
city_api._CITY_DETAIL_BATCH_RESPONSE_CACHE.clear()
city_api._CITY_DETAIL_BATCH_RESPONSE_CACHE_TS.clear()
city_api._CITY_DETAIL_BATCH_RESPONSE_INFLIGHT.clear()
monkeypatch.setenv("POLYWEATHER_CITY_DETAIL_BATCH_RESPONSE_CACHE_TTL_SEC", "20")
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", assert_entitlement)
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api, "_build_city_detail_batch_item_async", build_batch_item)
first = asyncio.run(
city_api.get_city_detail_batch_payload(
object(),
cities="Paris",
resolution="10m",
limit=12,
)
)
second = asyncio.run(
city_api.get_city_detail_batch_payload(
object(),
cities="Paris",
resolution="10m",
limit=12,
)
)
assert first == second
assert first["details"]["paris"]["resolution"] == "10m"
assert entitlement_calls == 2
assert build_calls == 1
def test_concurrent_city_detail_batch_requests_share_inflight_response(monkeypatch):
import asyncio
entitlement_calls = 0
build_calls = 0
async def build_batch_item(city, **kwargs):
nonlocal build_calls
build_calls += 1
await asyncio.sleep(0.02)
return city, {
"city": city,
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [20.0]},
"resolution": kwargs.get("resolution"),
}
def assert_entitlement(request):
nonlocal entitlement_calls
entitlement_calls += 1
city_api._CITY_DETAIL_BATCH_RESPONSE_CACHE.clear()
city_api._CITY_DETAIL_BATCH_RESPONSE_CACHE_TS.clear()
city_api._CITY_DETAIL_BATCH_RESPONSE_INFLIGHT.clear()
monkeypatch.setenv("POLYWEATHER_CITY_DETAIL_BATCH_RESPONSE_CACHE_TTL_SEC", "20")
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", assert_entitlement)
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api, "_build_city_detail_batch_item_async", build_batch_item)
async def run_requests():
return await asyncio.gather(
city_api.get_city_detail_batch_payload(
object(),
cities="Paris",
resolution="10m",
limit=12,
),
city_api.get_city_detail_batch_payload(
object(),
cities="Paris",
resolution="10m",
limit=12,
),
)
first, second = asyncio.run(run_requests())
assert first == second
assert entitlement_calls == 2
assert build_calls == 1
def test_concurrent_city_detail_requests_share_same_full_cache_refresh(monkeypatch): def test_concurrent_city_detail_requests_share_same_full_cache_refresh(monkeypatch):
import asyncio import asyncio
@@ -678,6 +1064,70 @@ def test_concurrent_city_detail_requests_share_same_full_cache_refresh(monkeypat
assert build_calls == 1 assert build_calls == 1
def test_stale_city_detail_uses_cached_full_payload_while_refreshing(monkeypatch):
import asyncio
refresh_calls = 0
build_inputs = []
class FakeCache:
def get_city_cache(self, kind, city):
assert kind == "full"
assert city == "paris"
return {
"payload": {
"city": "paris",
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [20.0]},
},
}
async def fake_run_in_threadpool(fn, *args, **kwargs):
if fn is city_api.legacy_routes._refresh_city_full_cache:
await asyncio.sleep(0.01)
return fn(*args, **kwargs)
def refresh_full(city, force_refresh):
nonlocal refresh_calls
refresh_calls += 1
return {
"city": city,
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [21.0]},
}
def build_detail(data, market_slug, target_date, resolution):
build_inputs.append(data["hourly"]["temps"][0])
return {
"city": data["city"],
"live_temp": data["hourly"]["temps"][0],
"resolution": resolution,
}
city_api._CITY_FULL_REFRESH_INFLIGHT.clear()
city_api._CITY_DETAIL_PAYLOAD_CACHE.clear()
city_api._CITY_DETAIL_PAYLOAD_CACHE_TS.clear()
city_api._CITY_DETAIL_PAYLOAD_INFLIGHT.clear()
monkeypatch.setattr(city_api, "run_in_threadpool", fake_run_in_threadpool)
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeCache())
monkeypatch.setattr(city_api.legacy_routes, "_city_cache_is_fresh", lambda entry, ttl: False)
monkeypatch.setattr(city_api.legacy_routes, "_overlay_latest_wunderground_current", lambda city, payload: payload)
monkeypatch.setattr(city_api.legacy_routes, "_refresh_city_full_cache", refresh_full)
monkeypatch.setattr(city_api.legacy_routes, "_build_city_detail_payload", build_detail)
async def run_request():
payload = await city_api.get_city_detail_aggregate_payload(object(), "Paris", resolution="10m")
await asyncio.sleep(0.03)
return payload
result = asyncio.run(run_request())
assert result["live_temp"] == 20.0
assert build_inputs == [20.0]
assert refresh_calls == 1
def test_force_refresh_invalidates_short_city_detail_payload_cache(monkeypatch): def test_force_refresh_invalidates_short_city_detail_payload_cache(monkeypatch):
import asyncio import asyncio
@@ -1917,6 +2367,102 @@ def test_scan_terminal_service_returns_stale_payload_after_failed_refresh(monkey
assert stale["stale_reason"] == "upstream 504" assert stale["stale_reason"] == "upstream 504"
def test_scan_terminal_timeout_does_not_replace_better_cached_snapshot(monkeypatch):
import time
filters = {"scan_mode": "tradable", "limit": 5}
normalized_filters = scan_terminal_service._normalize_scan_terminal_filters(filters)
scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
previous_payload = {
"generated_at": "2026-05-31T00:00:00Z",
"snapshot_id": "scan-existing",
"filters": normalized_filters,
"summary": {"candidate_total": 2, "visible_count": 2},
"top_signal": {"id": "old-1"},
"rows": [{"id": "old-1"}, {"id": "old-2"}],
"status": "ready",
"stale": False,
"stale_reason": None,
"last_success_at": None,
"last_failed_at": None,
}
scan_terminal_cache.set_cached_scan_terminal_payload(
normalized_filters,
previous_payload,
)
monkeypatch.setattr(
scan_terminal_service,
"CITIES",
{"fast": {"tz": 0}, "slow": {"tz": 0}},
)
monkeypatch.setattr(scan_terminal_service, "SCAN_TERMINAL_BUILD_TIMEOUT_SEC", 0.01)
monkeypatch.setattr(scan_terminal_service, "SCAN_TERMINAL_MAX_WORKERS", 2)
def _scan_city(city_name, *_args, **_kwargs):
if city_name == "slow":
time.sleep(0.05)
return {
"city": city_name,
"candidate_total": 1,
"primary_scores": [80.0],
"rows": [
{
"id": f"{city_name}-row",
"market_key": f"{city_name}-market",
"edge_percent": 4.0,
"final_score": 80.0,
"volume": 1000,
}
],
}
monkeypatch.setattr(scan_terminal_service, "_scan_city_terminal_rows", _scan_city)
stale = scan_terminal_service.build_scan_terminal_payload(filters, force_refresh=True)
assert stale["status"] == "stale"
assert stale["stale"] is True
assert [row["id"] for row in stale["rows"]] == ["old-1", "old-2"]
assert stale["stale_reason"].startswith("scan terminal build timed out")
cached = scan_terminal_cache.get_cached_scan_terminal_payload(
normalized_filters,
ttl_sec=3600,
)
assert [row["id"] for row in cached["rows"]] == ["old-1", "old-2"]
def test_scan_terminal_prewarm_builds_default_terminal_payload(monkeypatch):
calls = []
def _fake_build(filters, *, force_refresh=False, timeout_sec=None):
calls.append((dict(filters), force_refresh, timeout_sec))
return {"rows": []}
monkeypatch.setattr(
scan_terminal_service,
"_build_scan_terminal_payload_uncached",
_fake_build,
)
assert scan_terminal_service._warm_scan_terminal_payloads() == 2
assert {filters["limit"] for filters, _, _ in calls} == {25, 180}
filters, force_refresh, timeout_sec = calls[0]
assert all(force_refresh is False for _, force_refresh, _ in calls)
assert all(
timeout_sec == scan_terminal_service.SCAN_TERMINAL_PREWARM_PAYLOAD_TIMEOUT_SEC
for _, _, timeout_sec in calls
)
assert filters["scan_mode"] == "tradable"
assert filters["min_price"] == 0.05
assert filters["max_price"] == 0.95
assert filters["min_edge_pct"] == 2.0
assert filters["min_liquidity"] == 500.0
assert filters["market_type"] == "maxtemp"
assert filters["time_range"] == "today"
assert filters["limit"] == 25
def test_scan_terminal_service_returns_failed_without_success_snapshot(monkeypatch): def test_scan_terminal_service_returns_failed_without_success_snapshot(monkeypatch):
filters = {"scan_mode": "tradable", "limit": 5} filters = {"scan_mode": "tradable", "limit": 5}
scan_terminal_cache._SCAN_TERMINAL_CACHE.clear() scan_terminal_cache._SCAN_TERMINAL_CACHE.clear()
+8
View File
@@ -37,6 +37,7 @@ from src.data_collection.city_registry import ALIASES, CITY_REGISTRY
from src.data_collection.city_time import get_city_utc_offset_seconds from src.data_collection.city_time import get_city_utc_offset_seconds
from src.database.runtime_state import IntradayPathSnapshotRepository from src.database.runtime_state import IntradayPathSnapshotRepository
from web.services.city_payloads import ( from web.services.city_payloads import (
build_city_chart_detail_payload as _city_chart_payload_detail,
build_city_detail_payload as _city_payload_detail, build_city_detail_payload as _city_payload_detail,
build_city_summary_payload as _city_payload_summary build_city_summary_payload as _city_payload_summary
) )
@@ -2312,6 +2313,13 @@ def _build_city_detail_payload(
) )
def _build_city_chart_detail_payload(
data: Dict[str, Any],
resolution: Optional[str] = "10m",
) -> Dict[str, Any]:
return _city_chart_payload_detail(data, resolution=resolution)
# ────────────────────────────────────────────────────────── # ──────────────────────────────────────────────────────────
# Routes # Routes
+6 -1
View File
@@ -24,10 +24,15 @@ from web.scan_terminal_service import start_scan_terminal_prewarm
_ROUTES_REGISTERED_FLAG = "_polyweather_routes_registered" _ROUTES_REGISTERED_FLAG = "_polyweather_routes_registered"
def _service_role() -> str:
return str(os.getenv("POLYWEATHER_SERVICE_ROLE") or "").strip().lower()
def _scan_terminal_prewarm_enabled() -> bool: def _scan_terminal_prewarm_enabled() -> bool:
return str( enabled = str(
os.getenv("POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED") or "false" os.getenv("POLYWEATHER_SCAN_TERMINAL_PREWARM_ENABLED") or "false"
).strip().lower() in {"1", "true", "yes", "on"} ).strip().lower() in {"1", "true", "yes", "on"}
return enabled and _service_role() in {"web", "api", "backend"}
def create_app() -> FastAPI: def create_app() -> FastAPI:
+1 -1
View File
@@ -511,7 +511,7 @@ def _is_excluded_model_name(model_name: str) -> bool:
def _sqlite_health() -> Dict[str, Any]: def _sqlite_health() -> Dict[str, Any]:
try: try:
with sqlite3.connect(_account_db.db_path) as conn: with sqlite3.connect(_account_db.db_path, timeout=0.05) as conn:
conn.execute("SELECT 1").fetchone() conn.execute("SELECT 1").fetchone()
return {"ok": True, "db_path": _account_db.db_path} return {"ok": True, "db_path": _account_db.db_path}
except Exception as exc: except Exception as exc:
+12 -3
View File
@@ -2,7 +2,7 @@
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from fastapi import APIRouter, BackgroundTasks, Query, Request from fastapi import APIRouter, BackgroundTasks, Query, Request, Response
from web.services.city_api import ( from web.services.city_api import (
get_city_detail_batch_payload, get_city_detail_batch_payload,
@@ -12,6 +12,7 @@ from web.services.city_api import (
list_cities_payload, list_cities_payload,
) )
from web.services.city_realtime_stream import get_realtime_stream_payload from web.services.city_realtime_stream import get_realtime_stream_payload
from web.services.request_timing import attach_server_timing_header
router = APIRouter(tags=["city"]) router = APIRouter(tags=["city"])
@@ -108,22 +109,27 @@ async def cities_model_range(
@router.get("/api/cities/detail-batch") @router.get("/api/cities/detail-batch")
async def city_detail_batch( async def city_detail_batch(
request: Request, request: Request,
response: Response,
cities: str = "", cities: str = "",
force_refresh: bool = False, force_refresh: bool = False,
market_slug: Optional[str] = None, market_slug: Optional[str] = None,
target_date: Optional[str] = None, target_date: Optional[str] = None,
resolution: Optional[str] = "10m", resolution: Optional[str] = "10m",
scope: Optional[str] = "full",
limit: int = 12, limit: int = 12,
): ):
return await get_city_detail_batch_payload( payload = await get_city_detail_batch_payload(
request, request,
cities=cities, cities=cities,
force_refresh=force_refresh, force_refresh=force_refresh,
market_slug=market_slug, market_slug=market_slug,
target_date=target_date, target_date=target_date,
resolution=resolution, resolution=resolution,
scope=scope,
limit=limit, limit=limit,
) )
attach_server_timing_header(response, request, "city_detail_batch_server_timing")
return payload
@router.get("/api/city/{name}") @router.get("/api/city/{name}")
@@ -159,13 +165,14 @@ async def city_summary(
@router.get("/api/city/{name}/detail") @router.get("/api/city/{name}/detail")
async def city_detail_aggregate( async def city_detail_aggregate(
request: Request, request: Request,
response: Response,
name: str, name: str,
force_refresh: bool = False, force_refresh: bool = False,
market_slug: Optional[str] = None, market_slug: Optional[str] = None,
target_date: Optional[str] = None, target_date: Optional[str] = None,
resolution: Optional[str] = "10m", resolution: Optional[str] = "10m",
): ):
return await get_city_detail_aggregate_payload( payload = await get_city_detail_aggregate_payload(
request, request,
name, name,
force_refresh=force_refresh, force_refresh=force_refresh,
@@ -173,6 +180,8 @@ async def city_detail_aggregate(
target_date=target_date, target_date=target_date,
resolution=resolution, resolution=resolution,
) )
attach_server_timing_header(response, request, "city_detail_server_timing")
return payload
@router.get("/api/city/{name}/realtime-stream") @router.get("/api/city/{name}/realtime-stream")
+24 -3
View File
@@ -1,6 +1,6 @@
"""Operations/admin API routes.""" """Operations/admin API routes."""
from fastapi import APIRouter, Request from fastapi import APIRouter, Request, Response
from web.core import GrantPointsRequest from web.core import GrantPointsRequest
from web.services.ops_api import ( from web.services.ops_api import (
@@ -30,14 +30,35 @@ from web.services.ops_api import (
get_ops_training_accuracy, get_ops_training_accuracy,
get_ops_telegram_audit, get_ops_telegram_audit,
) )
from web.services.request_timing import ServerTimingRecorder, attach_server_timing_header
router = APIRouter(tags=["ops"]) router = APIRouter(tags=["ops"])
@router.get("/api/ops/online-users") @router.get("/api/ops/online-users")
async def ops_online_users(): async def ops_online_users(request: Request, response: Response):
timer = ServerTimingRecorder(
request,
log_name="ops_online_users_timing",
prefix="ops_online_users",
state_attr="ops_online_users_server_timing",
)
outcome = "ok"
status_code = 200
try:
from src.utils.online_tracker import online_count from src.utils.online_tracker import online_count
return {"online": online_count()} return {"online": timer.measure("online_count", online_count)}
except Exception:
outcome = "exception"
status_code = 500
raise
finally:
timer.finish(outcome=outcome, status_code=status_code)
attach_server_timing_header(
response,
request,
"ops_online_users_server_timing",
)
@router.get("/api/ops/users") @router.get("/api/ops/users")
+4 -1
View File
@@ -9,6 +9,7 @@ from web.services.scan_api import (
get_scan_terminal_overview_payload, get_scan_terminal_overview_payload,
get_scan_terminal_payload, get_scan_terminal_payload,
) )
from web.services.request_timing import attach_server_timing_header
router = APIRouter(tags=["scan"]) router = APIRouter(tags=["scan"])
@@ -45,12 +46,14 @@ async def scan_terminal(
region=region or trading_region or None, region=region or trading_region or None,
timezone_offset_seconds=timezone_offset_seconds, timezone_offset_seconds=timezone_offset_seconds,
) )
return JSONResponse( response = JSONResponse(
content=payload, content=payload,
headers={ headers={
"Cache-Control": "public, s-maxage=30, stale-while-revalidate=120", "Cache-Control": "public, s-maxage=30, stale-while-revalidate=120",
}, },
) )
attach_server_timing_header(response, request, "scan_terminal_server_timing")
return response
@router.post("/api/scan/terminal/overview") @router.post("/api/scan/terminal/overview")
+44 -6
View File
@@ -19,6 +19,10 @@ router = APIRouter(tags=["events"])
event_store = create_realtime_event_store() event_store = create_realtime_event_store()
_live_subscription_lock = threading.Lock() _live_subscription_lock = threading.Lock()
_live_subscription_started = False _live_subscription_started = False
SSE_REPLAY_BASE_LIMIT = 60
SSE_REPLAY_EVENTS_PER_CITY = 24
SSE_REPLAY_MAX_LIMIT = 240
SSE_REPLAY_DIRECT_RESYNC_FACTOR = 3
def _parse_cities_param(cities: str) -> Set[str]: def _parse_cities_param(cities: str) -> Set[str]:
@@ -29,12 +33,33 @@ def _parse_cities_param(cities: str) -> Set[str]:
} }
def _bounded_replay_limit(value: int) -> int: def _recommended_replay_limit(city_count: int) -> int:
requested = max(1, int(city_count or 0)) * SSE_REPLAY_EVENTS_PER_CITY
return max(SSE_REPLAY_BASE_LIMIT, min(SSE_REPLAY_MAX_LIMIT, requested))
def _bounded_replay_limit(value: int, *, city_count: int = 0) -> int:
try: try:
limit = int(value) limit = int(value)
except (TypeError, ValueError): except (TypeError, ValueError):
limit = 500 limit = _recommended_replay_limit(city_count)
return max(1, min(MAX_REPLAY_LIMIT, limit)) route_limit = _recommended_replay_limit(city_count)
return max(1, min(MAX_REPLAY_LIMIT, route_limit, limit))
def _should_direct_resync(
*,
since_revision: int,
latest_revision: int,
limit: int,
) -> bool:
since = max(0, int(since_revision or 0))
latest = max(0, int(latest_revision or 0))
if since <= 0 or latest <= since:
return False
gap = latest - since
threshold = max(SSE_REPLAY_MAX_LIMIT, max(1, int(limit or 1)) * SSE_REPLAY_DIRECT_RESYNC_FACTOR)
return gap > threshold
def _ensure_live_subscription() -> None: def _ensure_live_subscription() -> None:
@@ -65,7 +90,7 @@ async def sse_events(
origin = request.headers.get("origin", "") origin = request.headers.get("origin", "")
allowed = origin in {"https://polyweather.top", "https://www.polyweather.top", "http://localhost:3000"} allowed = origin in {"https://polyweather.top", "https://www.polyweather.top", "http://localhost:3000"}
city_set = _parse_cities_param(cities) city_set = _parse_cities_param(cities)
limit = _bounded_replay_limit(replay_limit) limit = _bounded_replay_limit(replay_limit, city_count=len(city_set))
_ensure_live_subscription() _ensure_live_subscription()
latest_revision = event_store.latest_revision() latest_revision = event_store.latest_revision()
replay_events = [] replay_events = []
@@ -73,14 +98,27 @@ async def sse_events(
if since_revision is not None: if since_revision is not None:
try: try:
since = max(0, int(since_revision))
if _should_direct_resync(
since_revision=since,
latest_revision=latest_revision,
limit=limit,
):
resync_event = {
"type": "resync_required",
"reason": "replay_gap_too_large",
"latest_revision": latest_revision,
"ts": int(time.time() * 1000),
}
else:
replay_events = event_store.replay_events( replay_events = event_store.replay_events(
cities=city_set, cities=city_set,
since_revision=max(0, int(since_revision)), since_revision=since,
limit=limit, limit=limit,
) )
if event_store.replay_requires_resync( if event_store.replay_requires_resync(
cities=city_set, cities=city_set,
since_revision=max(0, int(since_revision)), since_revision=since,
replay_count=len(replay_events), replay_count=len(replay_events),
limit=limit, limit=limit,
): ):
+116 -10
View File
@@ -1,6 +1,8 @@
from __future__ import annotations from __future__ import annotations
import json import json
import hashlib
import os
import threading import threading
import time import time
from datetime import datetime from datetime import datetime
@@ -9,21 +11,115 @@ from typing import Any, Dict, Optional
_SCAN_TERMINAL_CACHE_LOCK = threading.Lock() _SCAN_TERMINAL_CACHE_LOCK = threading.Lock()
_SCAN_TERMINAL_CACHE: Dict[str, Dict[str, Any]] = {} _SCAN_TERMINAL_CACHE: Dict[str, Dict[str, Any]] = {}
_SCAN_TERMINAL_REFRESHING: set[str] = set() _SCAN_TERMINAL_REFRESHING: set[str] = set()
_SCAN_TERMINAL_REDIS_CLIENT_LOCK = threading.Lock()
_SCAN_TERMINAL_REDIS_CLIENT: Any = None
_SCAN_TERMINAL_REDIS_UNAVAILABLE = False
def scan_terminal_cache_key(filters: Dict[str, Any]) -> str: def scan_terminal_cache_key(filters: Dict[str, Any]) -> str:
return json.dumps(filters, ensure_ascii=True, sort_keys=True) return json.dumps(filters, ensure_ascii=True, sort_keys=True)
def _truthy_env(name: str, *, default: bool = False) -> bool:
value = os.getenv(name)
if value is None:
return default
return str(value).strip().lower() not in ("", "0", "false", "no", "off")
def _redis_cache_enabled() -> bool:
return _truthy_env(
"POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_ENABLED",
default=bool(os.getenv("POLYWEATHER_REDIS_URL")),
)
def _redis_cache_ttl_sec() -> int:
try:
value = int(os.getenv("POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_TTL_SEC", "21600"))
except Exception:
value = 21600
return max(600, min(value, 86400))
def _redis_cache_prefix() -> str:
return os.getenv(
"POLYWEATHER_SCAN_TERMINAL_REDIS_CACHE_PREFIX",
"polyweather:scan_terminal:v1:",
)
def _redis_entry_key(cache_key: str) -> str:
digest = hashlib.sha256(cache_key.encode("utf-8")).hexdigest()
return f"{_redis_cache_prefix()}{digest}"
def _get_redis_client() -> Any:
global _SCAN_TERMINAL_REDIS_CLIENT, _SCAN_TERMINAL_REDIS_UNAVAILABLE
if not _redis_cache_enabled() or _SCAN_TERMINAL_REDIS_UNAVAILABLE:
return None
with _SCAN_TERMINAL_REDIS_CLIENT_LOCK:
if _SCAN_TERMINAL_REDIS_CLIENT is not None:
return _SCAN_TERMINAL_REDIS_CLIENT
try:
import redis # type: ignore
url = os.getenv("POLYWEATHER_REDIS_URL") or "redis://127.0.0.1:6379/0"
client = redis.Redis.from_url(
url,
socket_timeout=float(os.getenv("POLYWEATHER_REDIS_SOCKET_TIMEOUT_SECONDS", "2")),
socket_connect_timeout=float(
os.getenv("POLYWEATHER_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", "1")
),
health_check_interval=30,
)
client.ping()
_SCAN_TERMINAL_REDIS_CLIENT = client
return client
except Exception:
_SCAN_TERMINAL_REDIS_UNAVAILABLE = True
return None
def _read_redis_cache_entry(cache_key: str) -> Optional[Dict[str, Any]]:
client = _get_redis_client()
if client is None:
return None
try:
raw = client.get(_redis_entry_key(cache_key))
if not raw:
return None
if isinstance(raw, bytes):
raw = raw.decode("utf-8")
entry = json.loads(str(raw))
return dict(entry) if isinstance(entry, dict) else None
except Exception:
return None
def _write_redis_cache_entry(cache_key: str, entry: Dict[str, Any]) -> None:
client = _get_redis_client()
if client is None:
return
try:
client.setex(
_redis_entry_key(cache_key),
_redis_cache_ttl_sec(),
json.dumps(entry, ensure_ascii=False, separators=(",", ":")),
)
except Exception:
return
def get_cached_scan_terminal_payload( def get_cached_scan_terminal_payload(
filters: Dict[str, Any], filters: Dict[str, Any],
*, *,
ttl_sec: int, ttl_sec: int,
) -> Optional[Dict[str, Any]]: ) -> Optional[Dict[str, Any]]:
cache_key = scan_terminal_cache_key(filters)
now = time.time() now = time.time()
with _SCAN_TERMINAL_CACHE_LOCK: cached = get_scan_terminal_cache_entry(filters)
cached = _SCAN_TERMINAL_CACHE.get(cache_key)
if not cached: if not cached:
return None return None
cached_at = float(cached.get("t") or 0.0) cached_at = float(cached.get("t") or 0.0)
@@ -39,10 +135,17 @@ def get_scan_terminal_cache_entry(filters: Dict[str, Any]) -> Optional[Dict[str,
cache_key = scan_terminal_cache_key(filters) cache_key = scan_terminal_cache_key(filters)
with _SCAN_TERMINAL_CACHE_LOCK: with _SCAN_TERMINAL_CACHE_LOCK:
cached = _SCAN_TERMINAL_CACHE.get(cache_key) cached = _SCAN_TERMINAL_CACHE.get(cache_key)
if not isinstance(cached, dict): if isinstance(cached, dict):
return None
return dict(cached) return dict(cached)
redis_entry = _read_redis_cache_entry(cache_key)
if not redis_entry:
return None
with _SCAN_TERMINAL_CACHE_LOCK:
_SCAN_TERMINAL_CACHE[cache_key] = dict(redis_entry)
return dict(redis_entry)
def set_cached_scan_terminal_payload( def set_cached_scan_terminal_payload(
filters: Dict[str, Any], filters: Dict[str, Any],
@@ -51,8 +154,7 @@ def set_cached_scan_terminal_payload(
cache_key = scan_terminal_cache_key(filters) cache_key = scan_terminal_cache_key(filters)
existing = get_scan_terminal_cache_entry(filters) or {} existing = get_scan_terminal_cache_entry(filters) or {}
now = time.time() now = time.time()
with _SCAN_TERMINAL_CACHE_LOCK: entry = {
_SCAN_TERMINAL_CACHE[cache_key] = {
"t": now, "t": now,
"payload": dict(payload), "payload": dict(payload),
"success_t": now, "success_t": now,
@@ -60,6 +162,9 @@ def set_cached_scan_terminal_payload(
"last_error": existing.get("last_error"), "last_error": existing.get("last_error"),
"last_failed_at": existing.get("last_failed_at"), "last_failed_at": existing.get("last_failed_at"),
} }
with _SCAN_TERMINAL_CACHE_LOCK:
_SCAN_TERMINAL_CACHE[cache_key] = dict(entry)
_write_redis_cache_entry(cache_key, entry)
def set_scan_terminal_failure_state( def set_scan_terminal_failure_state(
@@ -68,11 +173,12 @@ def set_scan_terminal_failure_state(
error_message: str, error_message: str,
) -> None: ) -> None:
cache_key = scan_terminal_cache_key(filters) cache_key = scan_terminal_cache_key(filters)
with _SCAN_TERMINAL_CACHE_LOCK: existing = get_scan_terminal_cache_entry(filters) or {}
existing = _SCAN_TERMINAL_CACHE.get(cache_key) or {}
existing["last_error"] = error_message existing["last_error"] = error_message
existing["last_failed_at"] = datetime.utcnow().isoformat() + "Z" existing["last_failed_at"] = datetime.utcnow().isoformat() + "Z"
_SCAN_TERMINAL_CACHE[cache_key] = existing with _SCAN_TERMINAL_CACHE_LOCK:
_SCAN_TERMINAL_CACHE[cache_key] = dict(existing)
_write_redis_cache_entry(cache_key, existing)
def mark_scan_terminal_refreshing(filters: Dict[str, Any]) -> bool: def mark_scan_terminal_refreshing(filters: Dict[str, Any]) -> bool:
+22 -2
View File
@@ -11,6 +11,22 @@ from web.scan_terminal_filters import (
market_region_from_tz_offset as _market_region_from_tz_offset, market_region_from_tz_offset as _market_region_from_tz_offset,
safe_int as _safe_int, safe_int as _safe_int,
) )
from web.services.city_payloads import aggregate_runway_history
SCAN_ROW_RUNWAY_HISTORY_RESOLUTION = "10m"
SCAN_ROW_MAX_RUNWAY_POINTS = 144
def _compact_runway_plate_history_for_scan(raw_history: Any) -> Dict[str, List[Dict[str, Any]]]:
if not isinstance(raw_history, dict) or not raw_history:
return {}
compacted = aggregate_runway_history(raw_history, SCAN_ROW_RUNWAY_HISTORY_RESOLUTION)
return {
str(runway): points[-SCAN_ROW_MAX_RUNWAY_POINTS:]
for runway, points in compacted.items()
if isinstance(points, list) and points
}
def _resolve_time_range_dates(data: Dict[str, Any], time_range: str) -> List[str]: def _resolve_time_range_dates(data: Dict[str, Any], time_range: str) -> List[str]:
@@ -132,7 +148,9 @@ def _build_terminal_row(
"amos": data.get("amos") or None, "amos": data.get("amos") or None,
"top_buckets": scan.get("top_buckets") or [], "top_buckets": scan.get("top_buckets") or [],
"all_buckets": scan.get("all_buckets") or [], "all_buckets": scan.get("all_buckets") or [],
"runway_plate_history": data.get("runway_plate_history") or {}, "runway_plate_history": _compact_runway_plate_history_for_scan(
data.get("runway_plate_history")
),
} }
@@ -232,7 +250,9 @@ def _build_quick_row(
"is_primary_signal": True, "is_primary_signal": True,
"accepting_orders": False, "accepting_orders": False,
"row_id": row_id, "row_id": row_id,
"runway_plate_history": data.get("runway_plate_history") or {}, "runway_plate_history": _compact_runway_plate_history_for_scan(
data.get("runway_plate_history")
),
} }
# Compute a simple edge: model top probability vs neutral # Compute a simple edge: model top probability vs neutral
best_model_prob = max( best_model_prob = max(
+42
View File
@@ -5,6 +5,48 @@ import json
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS = 9
SCAN_PAYLOAD_DEFERRED_RUNWAY_POINTS = 12
def _compact_runway_history_points(
history: Any,
*,
max_points: int,
) -> Dict[str, List[Dict[str, Any]]]:
if not isinstance(history, dict) or max_points <= 0:
return {}
compacted: Dict[str, List[Dict[str, Any]]] = {}
for runway, points in history.items():
if not isinstance(points, list) or not points:
continue
compacted[str(runway)] = points[-max_points:]
return compacted
def compact_ranked_scan_rows_for_payload(
rows: List[Dict[str, Any]],
*,
full_history_rows: int = SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS,
deferred_runway_points: int = SCAN_PAYLOAD_DEFERRED_RUNWAY_POINTS,
) -> List[Dict[str, Any]]:
compacted_rows: List[Dict[str, Any]] = []
for index, row in enumerate(rows):
if index < full_history_rows:
compacted_rows.append(row)
continue
history = row.get("runway_plate_history")
if not isinstance(history, dict) or not history:
compacted_rows.append(row)
continue
next_row = dict(row)
next_row["runway_plate_history"] = _compact_runway_history_points(
history,
max_points=deferred_runway_points,
)
compacted_rows.append(next_row)
return compacted_rows
def build_scan_terminal_snapshot_id( def build_scan_terminal_snapshot_id(
filters: Dict[str, Any], filters: Dict[str, Any],
+138 -9
View File
@@ -16,6 +16,7 @@ from web.services.scan_terminal_config import (
SCAN_TERMINAL_BUILD_TIMEOUT_SEC, SCAN_TERMINAL_BUILD_TIMEOUT_SEC,
SCAN_TERMINAL_MAX_WORKERS, SCAN_TERMINAL_MAX_WORKERS,
SCAN_TERMINAL_PAYLOAD_TTL_SEC, SCAN_TERMINAL_PAYLOAD_TTL_SEC,
SCAN_TERMINAL_PREWARM_PAYLOAD_TIMEOUT_SEC,
) )
from src.data_collection.city_registry import ALIASES from src.data_collection.city_registry import ALIASES
from web.scan_terminal_cache import ( from web.scan_terminal_cache import (
@@ -34,6 +35,7 @@ from web.scan_terminal_payloads import (
build_failed_scan_terminal_payload, build_failed_scan_terminal_payload,
build_scan_terminal_snapshot_id, build_scan_terminal_snapshot_id,
build_stale_scan_terminal_payload, build_stale_scan_terminal_payload,
compact_ranked_scan_rows_for_payload,
) )
from web.scan_terminal_ranker import build_ranked_scan_terminal_result from web.scan_terminal_ranker import build_ranked_scan_terminal_result
def _normalize_locale(value: Any) -> str: def _normalize_locale(value: Any) -> str:
@@ -47,6 +49,35 @@ def _normalize_city_key(value: Any) -> str:
return ALIASES.get(text, text) return ALIASES.get(text, text)
def _rows_count(payload: Dict[str, Any]) -> int:
rows = payload.get("rows")
return len(rows) if isinstance(rows, list) else 0
def _build_stale_payload_for_timeout_if_better_cached(
*,
filters: Dict[str, Any],
cached_entry: Dict[str, Any],
ranked_rows: List[Dict[str, Any]],
timeout_message: Optional[str],
) -> Optional[Dict[str, Any]]:
success_payload = cached_entry.get("success_payload")
if not isinstance(success_payload, dict) or not success_payload.get("rows"):
return None
if _rows_count(success_payload) < len(ranked_rows):
return None
error_message = timeout_message or "市场扫描快照正在刷新中"
set_scan_terminal_failure_state(filters, error_message=error_message)
failed_entry = get_scan_terminal_cache_entry(filters) or cached_entry
return build_stale_scan_terminal_payload(
filters=filters,
success_payload=success_payload,
error_message=error_message,
failed_at=failed_entry.get("last_failed_at"),
)
def _start_scan_terminal_background_refresh(filters: Dict[str, Any]) -> bool: def _start_scan_terminal_background_refresh(filters: Dict[str, Any]) -> bool:
if not mark_scan_terminal_refreshing(filters): if not mark_scan_terminal_refreshing(filters):
return False return False
@@ -72,11 +103,17 @@ def _build_scan_terminal_payload_uncached(
filters: Dict[str, Any], filters: Dict[str, Any],
*, *,
force_refresh: bool = False, force_refresh: bool = False,
timeout_sec: Optional[float] = None,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
cached_entry = get_scan_terminal_cache_entry(filters) or {} cached_entry = get_scan_terminal_cache_entry(filters) or {}
try: try:
city_names = list(CITIES.keys()) city_names = list(CITIES.keys())
build_timeout_sec = float(
timeout_sec
if timeout_sec is not None
else SCAN_TERMINAL_BUILD_TIMEOUT_SEC
)
timezone_offset = filters.get("timezone_offset_seconds") timezone_offset = filters.get("timezone_offset_seconds")
if timezone_offset is not None: if timezone_offset is not None:
target_tz = int(timezone_offset) target_tz = int(timezone_offset)
@@ -116,7 +153,7 @@ def _build_scan_terminal_payload_uncached(
try: try:
completed = as_completed( completed = as_completed(
future_map, future_map,
timeout=float(SCAN_TERMINAL_BUILD_TIMEOUT_SEC), timeout=build_timeout_sec,
) )
for future in completed: for future in completed:
city_name = future_map[future] city_name = future_map[future]
@@ -131,8 +168,7 @@ def _build_scan_terminal_payload_uncached(
except FutureTimeoutError: except FutureTimeoutError:
timed_out = True timed_out = True
timeout_message = ( timeout_message = (
f"scan terminal build timed out after " f"scan terminal build timed out after {build_timeout_sec:g}s"
f"{SCAN_TERMINAL_BUILD_TIMEOUT_SEC}s"
) )
failed_reasons.append(timeout_message) failed_reasons.append(timeout_message)
for future, city_name in future_map.items(): for future, city_name in future_map.items():
@@ -175,7 +211,9 @@ def _build_scan_terminal_payload_uncached(
total_city_count=len(city_names), total_city_count=len(city_names),
failed_city_count=len(failed_cities), failed_city_count=len(failed_cities),
) )
ranked_rows = ranked_result["ranked_rows"] ranked_rows = compact_ranked_scan_rows_for_payload(
ranked_result["ranked_rows"]
)
if timed_out and not ranked_rows: if timed_out and not ranked_rows:
success_payload = cached_entry.get("success_payload") success_payload = cached_entry.get("success_payload")
@@ -189,6 +227,16 @@ def _build_scan_terminal_payload_uncached(
summary = ranked_result["summary"] summary = ranked_result["summary"]
top_signal = ranked_result["top_signal"] top_signal = ranked_result["top_signal"]
if timed_out:
stale_payload = _build_stale_payload_for_timeout_if_better_cached(
filters=filters,
cached_entry=cached_entry,
ranked_rows=ranked_rows,
timeout_message=timeout_message,
)
if stale_payload is not None:
return stale_payload
payload = { payload = {
"generated_at": datetime.utcnow().isoformat() + "Z", "generated_at": datetime.utcnow().isoformat() + "Z",
"filters": filters, "filters": filters,
@@ -235,16 +283,41 @@ def build_scan_terminal_payload(
raw_filters: Optional[Dict[str, Any]] = None, raw_filters: Optional[Dict[str, Any]] = None,
*, *,
force_refresh: bool = False, force_refresh: bool = False,
timing_recorder: Any = None,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
filters = _normalize_scan_terminal_filters(raw_filters) filters = (
timing_recorder.measure(
"normalize_filters",
lambda: _normalize_scan_terminal_filters(raw_filters),
)
if timing_recorder is not None
else _normalize_scan_terminal_filters(raw_filters)
)
if not force_refresh: if not force_refresh:
cached = get_cached_scan_terminal_payload( cached = (
timing_recorder.measure(
"cache_lookup",
lambda: get_cached_scan_terminal_payload(
filters,
ttl_sec=SCAN_TERMINAL_PAYLOAD_TTL_SEC,
),
)
if timing_recorder is not None
else get_cached_scan_terminal_payload(
filters, ttl_sec=SCAN_TERMINAL_PAYLOAD_TTL_SEC filters, ttl_sec=SCAN_TERMINAL_PAYLOAD_TTL_SEC
) )
)
if cached is not None: if cached is not None:
return cached return cached
cached_entry = get_scan_terminal_cache_entry(filters) or {} cached_entry = (
timing_recorder.measure(
"stale_cache_lookup",
lambda: get_scan_terminal_cache_entry(filters) or {},
)
if timing_recorder is not None
else get_scan_terminal_cache_entry(filters) or {}
)
success_payload = cached_entry.get("success_payload") success_payload = cached_entry.get("success_payload")
if isinstance(success_payload, dict) and success_payload: if isinstance(success_payload, dict) and success_payload:
started = _start_scan_terminal_background_refresh(filters) started = _start_scan_terminal_background_refresh(filters)
@@ -257,11 +330,65 @@ def build_scan_terminal_payload(
failed_at=cached_entry.get("last_failed_at"), failed_at=cached_entry.get("last_failed_at"),
) )
return _build_scan_terminal_payload_uncached(filters, force_refresh=force_refresh) return (
timing_recorder.measure(
"uncached_build",
lambda: _build_scan_terminal_payload_uncached(
filters,
force_refresh=force_refresh,
),
)
if timing_recorder is not None
else _build_scan_terminal_payload_uncached(filters, force_refresh=force_refresh)
)
_SCAN_PREWARM_STARTED = False _SCAN_PREWARM_STARTED = False
_SCAN_PREWARM_LOCK = threading.Lock() _SCAN_PREWARM_LOCK = threading.Lock()
_SCAN_TERMINAL_PREWARM_RAW_FILTERS = [
{
"scan_mode": "tradable",
"min_price": 0.05,
"max_price": 0.95,
"min_edge_pct": 2,
"min_liquidity": 500,
"market_type": "maxtemp",
"time_range": "today",
"limit": 25,
},
{
"scan_mode": "tradable",
"min_price": 0.05,
"max_price": 0.95,
"min_edge_pct": 2,
"min_liquidity": 500,
"market_type": "maxtemp",
"time_range": "today",
"limit": 180,
}
]
def _scan_terminal_prewarm_filters() -> List[Dict[str, Any]]:
return [
_normalize_scan_terminal_filters(filters)
for filters in _SCAN_TERMINAL_PREWARM_RAW_FILTERS
]
def _warm_scan_terminal_payloads() -> int:
ok = 0
for filters in _scan_terminal_prewarm_filters():
try:
_build_scan_terminal_payload_uncached(
filters,
force_refresh=False,
timeout_sec=SCAN_TERMINAL_PREWARM_PAYLOAD_TIMEOUT_SEC,
)
ok += 1
except Exception as exc:
logger.warning("scan terminal payload pre-warm failed: {}", exc)
return ok
def start_scan_terminal_prewarm() -> None: def start_scan_terminal_prewarm() -> None:
@@ -305,11 +432,13 @@ def start_scan_terminal_prewarm() -> None:
ok += 1 ok += 1
except Exception: except Exception:
pass pass
payload_ok = _warm_scan_terminal_payloads()
elapsed = int(time.time() - started) elapsed = int(time.time() - started)
logger.info( logger.info(
"scan terminal pre-warm finished ok={}/{} elapsed={}s", "scan terminal pre-warm finished ok={}/{} payloads={} elapsed={}s",
ok, ok,
len(city_names), len(city_names),
payload_ok,
elapsed, elapsed,
) )
except (ValueError, OSError, IOError): except (ValueError, OSError, IOError):
+429 -19
View File
@@ -13,6 +13,7 @@ from fastapi.concurrency import run_in_threadpool
from loguru import logger from loguru import logger
import web.routes as legacy_routes import web.routes as legacy_routes
from web.services.request_timing import ServerTimingRecorder
_RECENT_DEB_CACHE: Optional[Dict[str, Dict[str, object]]] = None _RECENT_DEB_CACHE: Optional[Dict[str, Dict[str, object]]] = None
_RECENT_DEB_CACHE_TS = 0.0 _RECENT_DEB_CACHE_TS = 0.0
@@ -23,13 +24,23 @@ _RECENT_DEB_CACHE_TTL_SEC = max(
int(os.getenv("POLYWEATHER_CITIES_DEB_RECENT_CACHE_TTL_SEC", "300") or "300"), int(os.getenv("POLYWEATHER_CITIES_DEB_RECENT_CACHE_TTL_SEC", "300") or "300"),
) )
_CITY_FULL_REFRESH_INFLIGHT: Dict[str, "asyncio.Task[Dict[str, Any]]"] = {} _CITY_FULL_REFRESH_INFLIGHT: Dict[str, "asyncio.Task[Dict[str, Any]]"] = {}
_CITY_FULL_STALE_REFRESH_TASKS: Dict[str, "asyncio.Task[Dict[str, Any]]"] = {}
_CITY_FULL_REFRESH_LOCK = asyncio.Lock() _CITY_FULL_REFRESH_LOCK = asyncio.Lock()
CityDetailPayloadCacheKey = Tuple[str, str, str, str, str, int] CityDetailPayloadCacheKey = Tuple[str, str, str, str, str, int]
CityChartDetailPayloadCacheKey = Tuple[str, str, str, int]
CityDetailBatchResponseCacheKey = Tuple[Tuple[str, ...], bool, str, str, str, str]
_CITY_DETAIL_PAYLOAD_CACHE: Dict[CityDetailPayloadCacheKey, Dict[str, Any]] = {} _CITY_DETAIL_PAYLOAD_CACHE: Dict[CityDetailPayloadCacheKey, Dict[str, Any]] = {}
_CITY_DETAIL_PAYLOAD_CACHE_TS: Dict[CityDetailPayloadCacheKey, float] = {} _CITY_DETAIL_PAYLOAD_CACHE_TS: Dict[CityDetailPayloadCacheKey, float] = {}
_CITY_DETAIL_PAYLOAD_INFLIGHT: Dict[CityDetailPayloadCacheKey, "asyncio.Task[Dict[str, Any]]"] = {} _CITY_DETAIL_PAYLOAD_INFLIGHT: Dict[CityDetailPayloadCacheKey, "asyncio.Task[Dict[str, Any]]"] = {}
_CITY_DETAIL_PAYLOAD_EPOCH: Dict[str, int] = {} _CITY_DETAIL_PAYLOAD_EPOCH: Dict[str, int] = {}
_CITY_DETAIL_PAYLOAD_LOCK = asyncio.Lock() _CITY_DETAIL_PAYLOAD_LOCK = asyncio.Lock()
_CITY_CHART_DETAIL_PAYLOAD_CACHE: Dict[CityChartDetailPayloadCacheKey, Dict[str, Any]] = {}
_CITY_CHART_DETAIL_PAYLOAD_CACHE_TS: Dict[CityChartDetailPayloadCacheKey, float] = {}
_CITY_CHART_DETAIL_PAYLOAD_LOCK = asyncio.Lock()
_CITY_DETAIL_BATCH_RESPONSE_CACHE: Dict[CityDetailBatchResponseCacheKey, Dict[str, Any]] = {}
_CITY_DETAIL_BATCH_RESPONSE_CACHE_TS: Dict[CityDetailBatchResponseCacheKey, float] = {}
_CITY_DETAIL_BATCH_RESPONSE_INFLIGHT: Dict[CityDetailBatchResponseCacheKey, "asyncio.Task[Dict[str, Any]]"] = {}
_CITY_DETAIL_BATCH_RESPONSE_LOCK = asyncio.Lock()
def _city_detail_payload_cache_ttl() -> float: def _city_detail_payload_cache_ttl() -> float:
@@ -40,6 +51,17 @@ def _city_detail_payload_cache_ttl() -> float:
return max(0.0, min(30.0, value)) return max(0.0, min(30.0, value))
def _city_detail_batch_response_cache_ttl() -> float:
try:
value = float(
os.getenv("POLYWEATHER_CITY_DETAIL_BATCH_RESPONSE_CACHE_TTL_SEC", "12")
or "12"
)
except ValueError:
value = 12.0
return max(0.0, min(30.0, value))
async def _overlay_cached_wunderground(city: str, payload: Dict[str, Any]) -> Dict[str, Any]: async def _overlay_cached_wunderground(city: str, payload: Dict[str, Any]) -> Dict[str, Any]:
return await run_in_threadpool( return await run_in_threadpool(
legacy_routes._overlay_latest_wunderground_current, legacy_routes._overlay_latest_wunderground_current,
@@ -53,9 +75,17 @@ async def _refresh_city_full_cache_singleflight(city: str, force_refresh: bool)
async with _CITY_FULL_REFRESH_LOCK: async with _CITY_FULL_REFRESH_LOCK:
task = _CITY_FULL_REFRESH_INFLIGHT.get(key) task = _CITY_FULL_REFRESH_INFLIGHT.get(key)
if task is None: if task is None:
task = asyncio.create_task( async def _run_refresh() -> Dict[str, Any]:
run_in_threadpool(legacy_routes._refresh_city_full_cache, city, force_refresh), try:
return await run_in_threadpool(
legacy_routes._refresh_city_full_cache,
city,
force_refresh,
) )
finally:
await _invalidate_city_detail_payload_cache(city)
task = asyncio.create_task(_run_refresh())
_CITY_FULL_REFRESH_INFLIGHT[key] = task _CITY_FULL_REFRESH_INFLIGHT[key] = task
try: try:
return await task return await task
@@ -76,6 +106,11 @@ async def _invalidate_city_detail_payload_cache(city: str) -> None:
for key in old_keys: for key in old_keys:
_CITY_DETAIL_PAYLOAD_CACHE.pop(key, None) _CITY_DETAIL_PAYLOAD_CACHE.pop(key, None)
_CITY_DETAIL_PAYLOAD_CACHE_TS.pop(key, None) _CITY_DETAIL_PAYLOAD_CACHE_TS.pop(key, None)
async with _CITY_CHART_DETAIL_PAYLOAD_LOCK:
old_chart_keys = [key for key in _CITY_CHART_DETAIL_PAYLOAD_CACHE if key[0] == normalized]
for key in old_chart_keys:
_CITY_CHART_DETAIL_PAYLOAD_CACHE.pop(key, None)
_CITY_CHART_DETAIL_PAYLOAD_CACHE_TS.pop(key, None)
async def _refresh_city_full_data(city: str, force_refresh: bool) -> Dict[str, Any]: async def _refresh_city_full_data(city: str, force_refresh: bool) -> Dict[str, Any]:
@@ -83,17 +118,61 @@ async def _refresh_city_full_data(city: str, force_refresh: bool) -> Dict[str, A
return await _refresh_city_full_cache_singleflight(city, force_refresh) return await _refresh_city_full_cache_singleflight(city, force_refresh)
def _start_city_full_stale_refresh(city: str) -> None:
normalized = str(city or "").strip().lower()
if not normalized:
return
existing = _CITY_FULL_STALE_REFRESH_TASKS.get(normalized)
if existing is not None and not existing.done():
return
task = asyncio.create_task(_refresh_city_full_data(city, False))
_CITY_FULL_STALE_REFRESH_TASKS[normalized] = task
def _cleanup(done: "asyncio.Task[Dict[str, Any]]") -> None:
if _CITY_FULL_STALE_REFRESH_TASKS.get(normalized) is done:
_CITY_FULL_STALE_REFRESH_TASKS.pop(normalized, None)
try:
done.result()
except Exception as exc: # pragma: no cover - defensive background guard
logger.warning("city full stale refresh failed city={}: {}", city, exc)
task.add_done_callback(_cleanup)
async def _get_city_full_data(city: str, *, force_refresh: bool) -> Dict[str, Any]: async def _get_city_full_data(city: str, *, force_refresh: bool) -> Dict[str, Any]:
if force_refresh: if force_refresh:
return await _refresh_city_full_data(city, True) return await _refresh_city_full_data(city, True)
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "full", city) cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "full", city)
if cached_entry: if cached_entry:
payload = cached_entry.get("payload") or {}
if not legacy_routes._city_cache_is_fresh(cached_entry, legacy_routes.CITY_FULL_CACHE_TTL_SEC): if not legacy_routes._city_cache_is_fresh(cached_entry, legacy_routes.CITY_FULL_CACHE_TTL_SEC):
if payload:
_start_city_full_stale_refresh(city)
return await _overlay_cached_wunderground(city, payload)
return await _refresh_city_full_data(city, False) return await _refresh_city_full_data(city, False)
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {}) return await _overlay_cached_wunderground(city, payload)
return await _refresh_city_full_data(city, False) return await _refresh_city_full_data(city, False)
async def _get_city_chart_data(city: str, *, force_refresh: bool) -> Dict[str, Any]:
if force_refresh:
return await _get_city_full_data(city, force_refresh=True)
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "full", city)
if cached_entry:
payload = cached_entry.get("payload") or {}
if payload:
if not legacy_routes._city_cache_is_fresh(cached_entry, legacy_routes.CITY_FULL_CACHE_TTL_SEC):
_start_city_full_stale_refresh(city)
return await _overlay_cached_wunderground(city, payload)
return {
"name": city,
"display_name": str((legacy_routes.CITY_REGISTRY.get(city, {}) or {}).get("display_name") or city.title()),
}
def _city_detail_payload_cache_key( def _city_detail_payload_cache_key(
data: Dict[str, Any], data: Dict[str, Any],
market_slug: Optional[str], market_slug: Optional[str],
@@ -119,6 +198,27 @@ def _city_detail_payload_cache_key(
) )
def _city_chart_detail_payload_cache_key(
data: Dict[str, Any],
resolution: Optional[str],
) -> CityChartDetailPayloadCacheKey:
city = str(data.get("city") or data.get("name") or "").strip().lower()
fingerprint = str(
data.get("updated_at_ts")
or data.get("updated_at")
or data.get("local_time")
or data.get("local_date")
or id(data)
)
generation = _CITY_DETAIL_PAYLOAD_EPOCH.get(city, 0)
return (
city,
str(resolution or "10m"),
fingerprint,
generation,
)
async def _build_city_detail_payload_cached( async def _build_city_detail_payload_cached(
data: Dict[str, Any], data: Dict[str, Any],
market_slug: Optional[str], market_slug: Optional[str],
@@ -176,6 +276,39 @@ async def _build_city_detail_payload_cached(
return payload return payload
async def _build_city_chart_detail_payload(
data: Dict[str, Any],
resolution: Optional[str],
) -> Dict[str, Any]:
ttl = _city_detail_payload_cache_ttl()
if ttl <= 0:
return legacy_routes._build_city_chart_detail_payload(data, resolution)
key = _city_chart_detail_payload_cache_key(data, resolution)
now_ts = time.time()
async with _CITY_CHART_DETAIL_PAYLOAD_LOCK:
cached = _CITY_CHART_DETAIL_PAYLOAD_CACHE.get(key)
cached_ts = _CITY_CHART_DETAIL_PAYLOAD_CACHE_TS.get(key, 0.0)
if cached is not None and now_ts - cached_ts < ttl:
return cached
payload = legacy_routes._build_city_chart_detail_payload(data, resolution)
async with _CITY_CHART_DETAIL_PAYLOAD_LOCK:
_CITY_CHART_DETAIL_PAYLOAD_CACHE[key] = payload
_CITY_CHART_DETAIL_PAYLOAD_CACHE_TS[key] = time.time()
if len(_CITY_CHART_DETAIL_PAYLOAD_CACHE) > 256:
oldest_keys = sorted(
_CITY_CHART_DETAIL_PAYLOAD_CACHE_TS,
key=lambda item: _CITY_CHART_DETAIL_PAYLOAD_CACHE_TS.get(item, 0.0),
)[:64]
for old_key in oldest_keys:
_CITY_CHART_DETAIL_PAYLOAD_CACHE.pop(old_key, None)
_CITY_CHART_DETAIL_PAYLOAD_CACHE_TS.pop(old_key, None)
return payload
def _default_deb_recent() -> Dict[str, object]: def _default_deb_recent() -> Dict[str, object]:
return { return {
"tier": "other", "tier": "other",
@@ -368,16 +501,41 @@ async def get_city_detail_aggregate_payload(
target_date: Optional[str] = None, target_date: Optional[str] = None,
resolution: Optional[str] = "10m", resolution: Optional[str] = "10m",
) -> Dict[str, Any]: ) -> Dict[str, Any]:
legacy_routes._assert_entitlement(request) timer = ServerTimingRecorder(
city = legacy_routes._normalize_city_or_404(name) request,
data = await _get_city_full_data(city, force_refresh=force_refresh) log_name="city_detail_timing",
prefix="city_detail",
state_attr="city_detail_server_timing",
)
outcome = "ok"
status_code = 200
try:
timer.measure("assert_entitlement", lambda: legacy_routes._assert_entitlement(request))
city = timer.measure("normalize_city", lambda: legacy_routes._normalize_city_or_404(name))
data = await timer.measure_async(
"full_data",
lambda: _get_city_full_data(city, force_refresh=force_refresh),
)
return await _build_city_detail_payload_cached( return await timer.measure_async(
"detail_payload",
lambda: _build_city_detail_payload_cached(
data, data,
market_slug, market_slug,
target_date, target_date,
resolution, resolution,
),
) )
except HTTPException as exc:
outcome = f"http_{exc.status_code}"
status_code = exc.status_code
raise
except Exception:
outcome = "exception"
status_code = 500
raise
finally:
timer.finish(outcome=outcome, status_code=status_code)
def _parse_batch_city_names(raw_cities: str, *, limit: int) -> List[str]: def _parse_batch_city_names(raw_cities: str, *, limit: int) -> List[str]:
@@ -397,6 +555,99 @@ def _parse_batch_city_names(raw_cities: str, *, limit: int) -> List[str]:
return out return out
def _city_detail_batch_response_cache_key(
city_names: List[str],
*,
force_refresh: bool,
market_slug: Optional[str],
target_date: Optional[str],
resolution: Optional[str],
scope: str,
) -> CityDetailBatchResponseCacheKey:
return (
tuple(city_names),
bool(force_refresh),
str(market_slug or ""),
str(target_date or ""),
str(resolution or "10m"),
str(scope or "full"),
)
def _normalize_city_detail_scope(scope: Optional[str]) -> str:
raw = str(scope or "full").strip().lower()
if raw in {"chart", "charts", "terminal", "terminal_chart"}:
return "chart"
return "full"
def _chart_scoped_city_detail(detail: Dict[str, Any]) -> Dict[str, Any]:
overview = detail.get("overview") if isinstance(detail.get("overview"), dict) else {}
timeseries = detail.get("timeseries") if isinstance(detail.get("timeseries"), dict) else {}
forecast = detail.get("forecast") if isinstance(detail.get("forecast"), dict) else {}
airport_primary_today_obs = (
detail.get("airport_primary_today_obs")
or overview.get("airport_primary_today_obs")
or []
)
forecast_daily = (
(forecast.get("daily") if isinstance(forecast, dict) else None)
or timeseries.get("forecast_daily")
or []
)
local_date = detail.get("local_date") or overview.get("local_date")
local_time = detail.get("local_time") or overview.get("local_time")
scoped = {
"city": detail.get("city") or overview.get("name"),
"fetched_at": detail.get("fetched_at"),
"local_date": local_date,
"local_time": local_time,
"overview": {
"name": overview.get("name"),
"display_name": overview.get("display_name"),
"local_date": local_date,
"local_time": local_time,
"temp_symbol": overview.get("temp_symbol"),
"current_temp": overview.get("current_temp"),
"deb_prediction": overview.get("deb_prediction"),
"settlement_source": overview.get("settlement_source"),
"settlement_source_label": overview.get("settlement_source_label"),
},
"timeseries": {
"hourly": timeseries.get("hourly") or detail.get("hourly") or {},
"metar_today_obs": timeseries.get("metar_today_obs") or [],
"settlement_today_obs": timeseries.get("settlement_today_obs") or [],
"forecast_daily": forecast_daily,
},
"hourly": timeseries.get("hourly") or detail.get("hourly") or {},
"models_hourly": detail.get("models_hourly") or {},
"deb": detail.get("deb") or {},
"forecast": {
"today_high": forecast.get("today_high") if isinstance(forecast, dict) else None,
"daily": forecast_daily,
},
"multi_model_daily": detail.get("multi_model_daily") or {},
"probabilities": detail.get("probabilities") or {"mu": None, "distribution": []},
"runway_plate_history": detail.get("runway_plate_history") or {},
"runway_band_history": detail.get("runway_band_history") or [],
"amos": detail.get("amos") or {},
"airport_current": detail.get("airport_current") or {},
"airport_primary": detail.get("airport_primary") or overview.get("airport_primary") or {},
"airport_primary_today_obs": airport_primary_today_obs,
"official": {"airport_primary_today_obs": airport_primary_today_obs},
"wunderground_current": detail.get("wunderground_current") or {},
"settlement_station": detail.get("settlement_station") or overview.get("settlement_station") or {},
}
return scoped
def _apply_city_detail_scope(detail: Dict[str, Any], scope: str) -> Dict[str, Any]:
if scope == "chart":
return _chart_scoped_city_detail(detail)
return detail
async def _build_city_detail_batch_item_async( async def _build_city_detail_batch_item_async(
city: str, city: str,
*, *,
@@ -404,7 +655,39 @@ async def _build_city_detail_batch_item_async(
market_slug: Optional[str], market_slug: Optional[str],
target_date: Optional[str], target_date: Optional[str],
resolution: Optional[str], resolution: Optional[str],
detail_scope: str = "full",
timing_recorder: Optional[ServerTimingRecorder] = None,
) -> Tuple[str, Dict[str, Any]]: ) -> Tuple[str, Dict[str, Any]]:
if detail_scope == "chart":
if timing_recorder is not None:
data = await timing_recorder.measure_async(
f"chart_data_{city}",
lambda: _get_city_chart_data(city, force_refresh=force_refresh),
)
detail = await timing_recorder.measure_async(
f"chart_payload_{city}",
lambda: _build_city_chart_detail_payload(data, resolution),
)
else:
data = await _get_city_chart_data(city, force_refresh=force_refresh)
detail = await _build_city_chart_detail_payload(data, resolution)
return city, detail
if timing_recorder is not None:
data = await timing_recorder.measure_async(
f"full_data_{city}",
lambda: _get_city_full_data(city, force_refresh=force_refresh),
)
detail = await timing_recorder.measure_async(
f"detail_payload_{city}",
lambda: _build_city_detail_payload_cached(
data,
market_slug,
target_date,
resolution,
),
)
else:
data = await _get_city_full_data(city, force_refresh=force_refresh) data = await _get_city_full_data(city, force_refresh=force_refresh)
detail = await _build_city_detail_payload_cached( detail = await _build_city_detail_payload_cached(
data, data,
@@ -412,7 +695,7 @@ async def _build_city_detail_batch_item_async(
target_date, target_date,
resolution, resolution,
) )
return city, detail return city, _apply_city_detail_scope(detail, detail_scope)
def _city_detail_batch_concurrency() -> int: def _city_detail_batch_concurrency() -> int:
@@ -423,6 +706,19 @@ def _city_detail_batch_concurrency() -> int:
return max(1, min(6, value)) return max(1, min(6, value))
def _city_detail_batch_partial_timeout_seconds() -> Optional[float]:
try:
timeout_ms = int(
os.getenv("POLYWEATHER_CITY_DETAIL_BATCH_PARTIAL_TIMEOUT_MS", "8500")
or "8500"
)
except ValueError:
timeout_ms = 8500
if timeout_ms <= 0:
return None
return max(0.001, min(60.0, timeout_ms / 1000.0))
async def get_city_detail_batch_payload( async def get_city_detail_batch_payload(
request: Request, request: Request,
*, *,
@@ -431,13 +727,37 @@ async def get_city_detail_batch_payload(
market_slug: Optional[str] = None, market_slug: Optional[str] = None,
target_date: Optional[str] = None, target_date: Optional[str] = None,
resolution: Optional[str] = "10m", resolution: Optional[str] = "10m",
scope: Optional[str] = "full",
limit: int = 12, limit: int = 12,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
legacy_routes._assert_entitlement(request) timer = ServerTimingRecorder(
city_names = _parse_batch_city_names(cities, limit=max(1, min(24, int(limit or 12)))) request,
log_name="city_detail_batch_timing",
prefix="city_detail_batch",
state_attr="city_detail_batch_server_timing",
)
outcome = "ok"
status_code = 200
try:
timer.measure("assert_entitlement", lambda: legacy_routes._assert_entitlement(request))
city_names = timer.measure(
"parse_cities",
lambda: _parse_batch_city_names(
cities,
limit=max(1, min(24, int(limit or 12))),
),
)
if not city_names: if not city_names:
return {"cities": [], "details": {}, "errors": {}} return {
"cities": [],
"details": {},
"errors": {},
"missing": [],
"partial": False,
}
detail_scope = _normalize_city_detail_scope(scope)
async def _build_uncached_payload() -> Dict[str, Any]:
semaphore = asyncio.Semaphore(_city_detail_batch_concurrency()) semaphore = asyncio.Semaphore(_city_detail_batch_concurrency())
async def _build_with_limit(city: str) -> Tuple[str, Dict[str, Any]]: async def _build_with_limit(city: str) -> Tuple[str, Dict[str, Any]]:
@@ -448,24 +768,114 @@ async def get_city_detail_batch_payload(
market_slug=market_slug, market_slug=market_slug,
target_date=target_date, target_date=target_date,
resolution=resolution, resolution=resolution,
detail_scope=detail_scope,
timing_recorder=timer,
) )
tasks = [ task_by_city = {
_build_with_limit(city) city: asyncio.create_task(_build_with_limit(city))
for city in city_names for city in city_names
] }
results = await asyncio.gather(*tasks, return_exceptions=True) task_city_lookup = {task: city for city, task in task_by_city.items()}
done, pending = await timer.measure_async(
"build_details",
lambda: asyncio.wait(
task_by_city.values(),
timeout=_city_detail_batch_partial_timeout_seconds(),
),
)
details: Dict[str, Any] = {} details: Dict[str, Any] = {}
errors: Dict[str, str] = {} errors: Dict[str, str] = {}
for city, result in zip(city_names, results): missing: List[str] = []
if isinstance(result, Exception): for task in done:
errors[city] = str(result) city = task_city_lookup[task]
try:
result_city, payload = task.result()
except Exception as exc:
errors[city] = str(exc)
continue continue
result_city, payload = result
details[result_city] = payload details[result_city] = payload
for task in pending:
city = task_city_lookup[task]
missing.append(city)
task.cancel()
missing_set = set(missing)
missing = [city for city in city_names if city in missing_set]
return { return {
"cities": city_names, "cities": city_names,
"details": details, "details": details,
"errors": errors, "errors": errors,
"missing": missing,
"partial": bool(missing or errors),
} }
cache_ttl = _city_detail_batch_response_cache_ttl()
cache_key = _city_detail_batch_response_cache_key(
city_names,
force_refresh=force_refresh,
market_slug=market_slug,
target_date=target_date,
resolution=resolution,
scope=detail_scope,
)
if cache_ttl > 0 and not force_refresh:
now_ts = time.time()
async with _CITY_DETAIL_BATCH_RESPONSE_LOCK:
cached = _CITY_DETAIL_BATCH_RESPONSE_CACHE.get(cache_key)
cached_ts = _CITY_DETAIL_BATCH_RESPONSE_CACHE_TS.get(cache_key, 0.0)
if cached is not None and now_ts - cached_ts < cache_ttl:
outcome = "cache_hit"
return cached
task = _CITY_DETAIL_BATCH_RESPONSE_INFLIGHT.get(cache_key)
owner = False
if task is None:
owner = True
task = asyncio.create_task(_build_uncached_payload())
_CITY_DETAIL_BATCH_RESPONSE_INFLIGHT[cache_key] = task
try:
payload = await timer.measure_async(
"build_or_wait_cached_batch",
lambda: task,
)
finally:
if owner and task.done():
async with _CITY_DETAIL_BATCH_RESPONSE_LOCK:
if _CITY_DETAIL_BATCH_RESPONSE_INFLIGHT.get(cache_key) is task:
_CITY_DETAIL_BATCH_RESPONSE_INFLIGHT.pop(cache_key, None)
if payload.get("partial"):
outcome = "partial"
elif not owner:
outcome = "shared_inflight"
if owner:
async with _CITY_DETAIL_BATCH_RESPONSE_LOCK:
if not payload.get("partial"):
_CITY_DETAIL_BATCH_RESPONSE_CACHE[cache_key] = payload
_CITY_DETAIL_BATCH_RESPONSE_CACHE_TS[cache_key] = time.time()
if len(_CITY_DETAIL_BATCH_RESPONSE_CACHE) > 128:
oldest_keys = sorted(
_CITY_DETAIL_BATCH_RESPONSE_CACHE_TS,
key=lambda item: _CITY_DETAIL_BATCH_RESPONSE_CACHE_TS.get(item, 0.0),
)[:32]
for old_key in oldest_keys:
_CITY_DETAIL_BATCH_RESPONSE_CACHE.pop(old_key, None)
_CITY_DETAIL_BATCH_RESPONSE_CACHE_TS.pop(old_key, None)
return payload
payload = await _build_uncached_payload()
if payload.get("partial"):
outcome = "partial"
return payload
except HTTPException as exc:
outcome = f"http_{exc.status_code}"
status_code = exc.status_code
raise
except Exception:
outcome = "exception"
status_code = 500
raise
finally:
timer.finish(outcome=outcome, status_code=status_code)
+145
View File
@@ -161,6 +161,151 @@ def build_runway_band_history(raw_history: Dict[str, List[Dict[str, Any]]], reso
return band_history return band_history
def build_runway_chart_histories(
raw_history: Dict[str, List[Dict[str, Any]]],
resolution: str,
) -> tuple[Dict[str, List[Dict[str, Any]]], List[Dict[str, Any]]]:
if not raw_history:
return {}, []
try:
if resolution.endswith("m"):
minutes = int(resolution[:-1])
elif resolution.endswith("h"):
minutes = int(resolution[:-1]) * 60
else:
minutes = 10
except Exception:
minutes = 10
seconds = max(60, minutes * 60)
per_runway_buckets: Dict[str, Dict[int, List[float]]] = {}
all_buckets: Dict[int, List[float]] = {}
for rwy, points in raw_history.items():
if not points:
continue
rwy_key = str(rwy or "").strip()
if not rwy_key:
continue
runway_buckets = per_runway_buckets.setdefault(rwy_key, {})
for pt in points:
t_str = pt.get("time") or pt.get("timestamp")
temp = pt.get("temp") or pt.get("temp_c") or pt.get("value")
if temp is None or not isinstance(t_str, str):
continue
dt = _parse_time_val(t_str)
if not dt:
continue
try:
temp_value = float(temp)
except Exception:
continue
bucket_ts = (int(dt.timestamp()) // seconds) * seconds
runway_buckets.setdefault(bucket_ts, []).append(temp_value)
all_buckets.setdefault(bucket_ts, []).append(temp_value)
aggregated_history: Dict[str, List[Dict[str, Any]]] = {}
for rwy, buckets in per_runway_buckets.items():
bucket_points = []
for bucket_ts in sorted(buckets.keys()):
temps = buckets[bucket_ts]
if not temps:
continue
bucket_dt = datetime.fromtimestamp(bucket_ts, tz=timezone.utc)
bucket_points.append(
{
"time": bucket_dt.isoformat(),
"temp": round(temps[-1], 1),
}
)
if bucket_points:
aggregated_history[rwy] = bucket_points
band_history = []
for bucket_ts in sorted(all_buckets.keys()):
temps = all_buckets[bucket_ts]
if not temps:
continue
bucket_dt = datetime.fromtimestamp(bucket_ts, tz=timezone.utc)
band_history.append(
{
"time": bucket_dt.isoformat(),
"high_temp": round(max(temps), 1),
"low_temp": round(min(temps), 1),
"avg_temp": round(sum(temps) / len(temps), 1),
}
)
return aggregated_history, band_history
def build_city_chart_detail_payload(
data: Dict[str, Any],
resolution: Optional[str] = "10m",
) -> Dict[str, Any]:
forecast = data.get("forecast") if isinstance(data.get("forecast"), dict) else {}
current = data.get("current") if isinstance(data.get("current"), dict) else {}
multi_model = data.get("multi_model") if isinstance(data.get("multi_model"), dict) else {}
runway_history, runway_band_history = build_runway_chart_histories(
data.get("runway_plate_history") or {},
resolution or "10m",
)
airport_primary_today_obs = data.get("airport_primary_today_obs") or []
local_date = data.get("local_date")
local_time = data.get("local_time")
return {
"city": data.get("name") or data.get("city"),
"fetched_at": data.get("updated_at"),
"local_date": local_date,
"local_time": local_time,
"overview": {
"name": data.get("name") or data.get("city"),
"display_name": data.get("display_name"),
"local_date": local_date,
"local_time": local_time,
"temp_symbol": data.get("temp_symbol"),
"current_temp": current.get("temp"),
"deb_prediction": (data.get("deb") or {}).get("prediction"),
"settlement_source": current.get("settlement_source"),
"settlement_source_label": current.get("settlement_source_label"),
},
"timeseries": {
"hourly": data.get("hourly") or {},
"metar_today_obs": data.get("metar_today_obs") or [],
"settlement_today_obs": data.get("settlement_today_obs") or [],
"forecast_daily": forecast.get("daily") or [],
},
"hourly": data.get("hourly") or {},
"models_hourly": {
"times": multi_model.get("hourly_times") or [],
"curves": {
model: values
for model, values in (multi_model.get("hourly_forecasts") or {}).items()
if not _is_excluded_model_name(model)
},
},
"deb": data.get("deb") or {},
"forecast": {
"today_high": forecast.get("today_high"),
"daily": forecast.get("daily") or [],
},
"multi_model_daily": data.get("multi_model_daily") or {},
"probabilities": data.get("probabilities") or {"mu": None, "distribution": []},
"runway_plate_history": runway_history,
"runway_band_history": runway_band_history,
"amos": data.get("amos") or {},
"airport_current": data.get("airport_current") or {},
"airport_primary": data.get("airport_primary") or {},
"airport_primary_today_obs": airport_primary_today_obs,
"official": {"airport_primary_today_obs": airport_primary_today_obs},
"wunderground_current": data.get("wunderground_current") or {},
"settlement_station": data.get("settlement_station") or {},
}
def build_city_detail_payload( def build_city_detail_payload(
data: Dict[str, Any], data: Dict[str, Any],
market_slug: Optional[str] = None, market_slug: Optional[str] = None,
+1
View File
@@ -25,6 +25,7 @@ from src.utils.refresh_policy import OBSERVATION_REFRESH_SEC, SCAN_ROWS_REFRESH_
from web.analysis_service import ( from web.analysis_service import (
_analyze, _analyze,
_analyze_summary, _analyze_summary,
_build_city_chart_detail_payload, # noqa: F401 - compatibility export for chart detail batches
_build_city_detail_payload, # noqa: F401 - compatibility export for tests and transitional routers _build_city_detail_payload, # noqa: F401 - compatibility export for tests and transitional routers
_build_city_market_scan_payload, _build_city_market_scan_payload,
_build_city_summary_payload, _build_city_summary_payload,
+62 -3
View File
@@ -545,18 +545,41 @@ def get_ops_billing_risk(
"limit": str(max(safe_limit * 10, 500)), "limit": str(max(safe_limit * 10, 500)),
}, },
) )
subscription_rows = collect( trial_subscription_rows = collect(
"subscriptions", "subscriptions",
{ {
"select": ( "select": (
"id,user_id,plan_code,source,status,starts_at,expires_at," "id,user_id,plan_code,source,status,starts_at,expires_at,"
"created_at,updated_at" "created_at,updated_at"
), ),
"or": "(source.eq.signup_trial,plan_code.eq.signup_trial_3d,status.eq.active)", "or": "(source.eq.signup_trial,plan_code.eq.signup_trial_3d)",
"order": "created_at.desc", "order": "created_at.desc",
"limit": str(max(safe_limit * 20, 1000)), "limit": str(max(safe_limit * 20, 1000)),
}, },
) )
active_subscription_rows = collect(
"subscriptions",
{
"select": (
"id,user_id,plan_code,source,status,starts_at,expires_at,"
"created_at,updated_at"
),
"status": "eq.active",
"order": "created_at.desc",
"limit": str(max(safe_limit * 20, 1000)),
},
)
subscription_rows: List[Dict[str, Any]] = []
seen_subscription_keys: set[str] = set()
for row in [*trial_subscription_rows, *active_subscription_rows]:
key = str(row.get("id") or "").strip() or (
f"{row.get('user_id')}:{row.get('plan_code')}:{row.get('source')}:"
f"{row.get('starts_at')}:{row.get('expires_at')}"
)
if key in seen_subscription_keys:
continue
seen_subscription_keys.add(key)
subscription_rows.append(row)
entitlement_trial_events = collect( entitlement_trial_events = collect(
"entitlement_events", "entitlement_events",
{ {
@@ -751,6 +774,40 @@ def get_ops_billing_risk(
def normalize_user_key(value: Any) -> str: def normalize_user_key(value: Any) -> str:
return str(value or "").strip().lower() return str(value or "").strip().lower()
def analytics_payload(row: Dict[str, Any]) -> Dict[str, Any]:
payload = row.get("payload")
return payload if isinstance(payload, dict) else {}
def analytics_correlation_keys(row: Dict[str, Any]) -> set[str]:
payload = analytics_payload(row)
keys: set[str] = set()
user_id = normalize_user_key(row.get("user_id") or payload.get("user_id"))
client_id = str(row.get("client_id") or "").strip()
session_id = str(row.get("session_id") or "").strip()
if user_id:
keys.add(f"user:{user_id}")
if client_id:
keys.add(f"client:{client_id}")
if session_id:
keys.add(f"session:{session_id}")
return keys
signup_intent_keys: set[str] = set()
for row in events:
if str(row.get("event_type") or "").strip().lower() != "login_start":
continue
payload = analytics_payload(row)
mode = str(payload.get("mode") or payload.get("auth_mode") or "").strip().lower()
if mode == "signup":
signup_intent_keys.update(analytics_correlation_keys(row))
def has_signup_intent(row: Dict[str, Any]) -> bool:
payload = analytics_payload(row)
mode = str(payload.get("mode") or payload.get("auth_mode") or "").strip().lower()
if mode == "signup" or payload.get("signup_intent") is True:
return True
return bool(analytics_correlation_keys(row).intersection(signup_intent_keys))
trial_actor_keys = { trial_actor_keys = {
_app_analytics_actor_key(row) _app_analytics_actor_key(row)
for row in events for row in events
@@ -817,10 +874,12 @@ def get_ops_billing_risk(
) )
for row in signup_rows[:300]: for row in signup_rows[:300]:
if not has_signup_intent(row):
continue
actor_key = _app_analytics_actor_key(row) actor_key = _app_analytics_actor_key(row)
if actor_key in trial_actor_keys: if actor_key in trial_actor_keys:
continue continue
payload = row.get("payload") if isinstance(row.get("payload"), dict) else {} payload = analytics_payload(row)
signup_user_id = normalize_user_key(row.get("user_id") or payload.get("user_id")) signup_user_id = normalize_user_key(row.get("user_id") or payload.get("user_id"))
if not signup_user_id: if not signup_user_id:
continue continue
+87
View File
@@ -0,0 +1,87 @@
"""Small helpers for exposing request stage timings via Server-Timing."""
from __future__ import annotations
import re
import threading
import time
from typing import Awaitable, Callable, Dict, Optional, TypeVar
from fastapi import Request, Response
from loguru import logger
T = TypeVar("T")
class ServerTimingRecorder:
def __init__(
self,
request: Optional[Request],
*,
log_name: str,
prefix: str,
state_attr: str,
) -> None:
self.request = request
self.log_name = log_name
self.prefix = prefix
self.state_attr = state_attr
self.started = time.perf_counter()
self.timings_ms: Dict[str, float] = {}
self._lock = threading.Lock()
def _record(self, stage: str, started: float) -> None:
elapsed_ms = round((time.perf_counter() - started) * 1000.0, 1)
with self._lock:
self.timings_ms[stage] = elapsed_ms
def measure(self, stage: str, action: Callable[[], T]) -> T:
started = time.perf_counter()
try:
return action()
finally:
self._record(stage, started)
async def measure_async(self, stage: str, action: Callable[[], Awaitable[T]]) -> T:
started = time.perf_counter()
try:
return await action()
finally:
self._record(stage, started)
def server_timing_value(self) -> str:
with self._lock:
items = list(self.timings_ms.items())
return ", ".join(
f"{self._metric_name(stage)};dur={max(0.0, duration):.1f}"
for stage, duration in items
)
def finish(self, *, outcome: str, status_code: int) -> None:
self._record("total", self.started)
value = self.server_timing_value()
state = getattr(self.request, "state", None)
if state is not None:
setattr(state, self.state_attr, value)
logger.info(
"{} outcome={} status_code={} timings_ms={}",
self.log_name,
outcome,
status_code,
dict(self.timings_ms),
)
def _metric_name(self, stage: str) -> str:
raw = f"{self.prefix}_{stage}"
return re.sub(r"[^A-Za-z0-9_-]", "_", raw)
def attach_server_timing_header(
response: Response,
request: Request,
state_attr: str,
) -> None:
value = str(getattr(request.state, state_attr, "") or "").strip()
if value:
response.headers["Server-Timing"] = value
+42 -7
View File
@@ -2,12 +2,25 @@
from __future__ import annotations from __future__ import annotations
from inspect import Parameter, signature
from typing import Any, Dict from typing import Any, Dict
from fastapi import Request from fastapi import HTTPException, Request
from fastapi.concurrency import run_in_threadpool from fastapi.concurrency import run_in_threadpool
import web.routes as legacy_routes import web.routes as legacy_routes
from web.services.request_timing import ServerTimingRecorder
def _supports_timing_recorder(func: Any) -> bool:
try:
params = signature(func).parameters.values()
except (TypeError, ValueError):
return True
return any(
param.name == "timing_recorder" or param.kind == Parameter.VAR_KEYWORD
for param in params
)
async def get_scan_terminal_payload( async def get_scan_terminal_payload(
@@ -26,7 +39,16 @@ async def get_scan_terminal_payload(
region: str = "", region: str = "",
timezone_offset_seconds: int | None = None, timezone_offset_seconds: int | None = None,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
legacy_routes._assert_entitlement(request) timer = ServerTimingRecorder(
request,
log_name="scan_terminal_timing",
prefix="scan_terminal",
state_attr="scan_terminal_server_timing",
)
outcome = "ok"
status_code = 200
try:
timer.measure("assert_entitlement", lambda: legacy_routes._assert_entitlement(request))
filters: Dict[str, Any] = { filters: Dict[str, Any] = {
"scan_mode": scan_mode, "scan_mode": scan_mode,
"min_price": min_price, "min_price": min_price,
@@ -42,11 +64,24 @@ async def get_scan_terminal_payload(
filters["timezone_offset_seconds"] = timezone_offset_seconds filters["timezone_offset_seconds"] = timezone_offset_seconds
if region: if region:
filters["trading_region"] = region filters["trading_region"] = region
return await run_in_threadpool( async def build_payload():
legacy_routes.build_scan_terminal_payload, builder = legacy_routes.build_scan_terminal_payload
filters, kwargs: Dict[str, Any] = {"force_refresh": force_refresh}
force_refresh=force_refresh, if _supports_timing_recorder(builder):
) kwargs["timing_recorder"] = timer
return await run_in_threadpool(builder, filters, **kwargs)
return await timer.measure_async("build_payload", build_payload)
except HTTPException as exc:
outcome = f"http_{exc.status_code}"
status_code = exc.status_code
raise
except Exception:
outcome = "exception"
status_code = 500
raise
finally:
timer.finish(outcome=outcome, status_code=status_code)
async def get_scan_terminal_overview_payload(request: Request) -> Dict[str, Any]: async def get_scan_terminal_overview_payload(request: Request) -> Dict[str, Any]:
+7 -1
View File
@@ -31,10 +31,16 @@ SCAN_TERMINAL_PAYLOAD_TTL_SEC = min(
) )
SCAN_TERMINAL_BUILD_TIMEOUT_SEC = _env_int( SCAN_TERMINAL_BUILD_TIMEOUT_SEC = _env_int(
"POLYWEATHER_SCAN_TERMINAL_BUILD_TIMEOUT_SEC", "POLYWEATHER_SCAN_TERMINAL_BUILD_TIMEOUT_SEC",
30, 10,
min_value=8, min_value=8,
max_value=30, max_value=30,
) )
SCAN_TERMINAL_PREWARM_PAYLOAD_TIMEOUT_SEC = _env_int(
"POLYWEATHER_SCAN_TERMINAL_PREWARM_PAYLOAD_TIMEOUT_SEC",
30,
min_value=10,
max_value=120,
)
SCAN_TERMINAL_MAX_WORKERS = _env_int( SCAN_TERMINAL_MAX_WORKERS = _env_int(
"POLYWEATHER_SCAN_TERMINAL_MAX_WORKERS", "POLYWEATHER_SCAN_TERMINAL_MAX_WORKERS",
8, 8,
+2 -5
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import time import time
from typing import Any, Dict, Optional from typing import Any, Dict, Optional
from fastapi import BackgroundTasks, HTTPException, Request from fastapi import BackgroundTasks, Request
from fastapi.concurrency import run_in_threadpool from fastapi.concurrency import run_in_threadpool
from fastapi.responses import PlainTextResponse from fastapi.responses import PlainTextResponse
from loguru import logger from loguru import logger
@@ -16,10 +16,7 @@ import web.routes as legacy_routes
def get_health_payload() -> Dict[str, Any]: def get_health_payload() -> Dict[str, Any]:
payload = build_health_payload() return build_health_payload()
if payload.get("status") != "ok":
raise HTTPException(status_code=503, detail=payload)
return payload
async def get_system_status_payload() -> Dict[str, Any]: async def get_system_status_payload() -> Dict[str, Any]: