From ea5c6ec71ed3cb7539180209f96ada7fd8ff270e Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Mon, 25 May 2026 23:03:55 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B8=85=E7=90=86=E5=B8=82=E5=9C=BA=E6=A6=82?= =?UTF-8?q?=E8=A7=88=E6=AE=8B=E7=95=99=EF=BC=9A=E5=88=A0=E9=99=A4=20Market?= =?UTF-8?q?OverviewBanner+refresh=5Fpolicy=20=E4=B8=AD=E7=9A=84=20marketOv?= =?UTF-8?q?erview=20=E6=9D=A1=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 22 - .../LiveTemperatureThresholdChart.tsx | 3 +- .../MarketOverviewBanner.module.css | 178 --- .../scan-terminal/MarketOverviewBanner.tsx | 132 -- .../scan-terminal/RealtimeScrollChart.tsx | 3 +- .../__tests__/refreshCadencePolicy.test.ts | 48 + .../scan-terminal/scan-terminal-client.ts | 3 +- .../scan-terminal/use-scan-terminal-query.ts | 3 +- frontend/lib/refresh-policy.ts | 13 + frontend/lib/source-freshness.ts | 21 +- src/data_collection/weather_sources.py | 19 +- src/utils/refresh_policy.py | 9 + tests/test_refresh_policy.py | 35 + tests/test_scan_terminal_modules.py | 53 - web/analysis_service.py | 27 - web/routers/scan.py | 12 - web/scan_city_ai_fallback.py | 507 ------- web/scan_city_ai_helpers.py | 322 ----- web/scan_city_ai_prompt.py | 261 ---- web/scan_city_ai_provider.py | 156 --- web/scan_terminal_ai_compact.py | 466 ------- web/scan_terminal_ai_merge.py | 256 ---- web/scan_terminal_cache.py | 73 - web/scan_terminal_city_row.py | 119 +- web/scan_terminal_filters.py | 2 +- web/scan_terminal_metar_gate.py | 2 +- web/scan_terminal_ranker.py | 2 +- web/scan_terminal_service.py | 1187 +---------------- web/services/city_runtime.py | 66 +- web/services/market_overview_api.py | 275 ++-- web/services/scan_ai_config.py | 8 +- web/services/scan_api.py | 35 - 32 files changed, 438 insertions(+), 3880 deletions(-) delete mode 100644 frontend/components/dashboard/scan-terminal/MarketOverviewBanner.module.css delete mode 100644 frontend/components/dashboard/scan-terminal/MarketOverviewBanner.tsx create mode 100644 frontend/components/dashboard/scan-terminal/__tests__/refreshCadencePolicy.test.ts create mode 100644 frontend/lib/refresh-policy.ts create mode 100644 src/utils/refresh_policy.py create mode 100644 tests/test_refresh_policy.py delete mode 100644 web/scan_city_ai_fallback.py delete mode 100644 web/scan_city_ai_helpers.py delete mode 100644 web/scan_city_ai_prompt.py delete mode 100644 web/scan_city_ai_provider.py delete mode 100644 web/scan_terminal_ai_compact.py delete mode 100644 web/scan_terminal_ai_merge.py diff --git a/.env.example b/.env.example index a25e5bb6..aa178e18 100644 --- a/.env.example +++ b/.env.example @@ -134,28 +134,6 @@ KNMI_API_KEY= ######################################## # 8) Optional modules ######################################## - -# Optional OpenAI-compatible market scan review for Pro users -# Temporary default provider: MiMo via https://token-plan-cn.xiaomimimo.com/v1. -POLYWEATHER_SCAN_AI_ENABLED=false -POLYWEATHER_SCAN_AI_API_KEY= -POLYWEATHER_SCAN_AI_PROVIDER=mimo -POLYWEATHER_SCAN_AI_PROVIDER_LABEL=MiMo -POLYWEATHER_SCAN_AI_BASE_URL=https://token-plan-cn.xiaomimimo.com/v1 -POLYWEATHER_SCAN_AI_MODEL=mimo-v2.5-pro -POLYWEATHER_SCAN_CITY_AI_MODEL=mimo-v2.5-pro -# Backward-compatible legacy DeepSeek variables are still read if the generic -# POLYWEATHER_SCAN_AI_* variables are unset. -# POLYWEATHER_DEEPSEEK_API_KEY= -# POLYWEATHER_DEEPSEEK_BASE_URL=https://api.deepseek.com -POLYWEATHER_SCAN_AI_TIMEOUT_SEC=18 -POLYWEATHER_SCAN_CITY_AI_TIMEOUT_SEC=30 -POLYWEATHER_SCAN_CITY_AI_RETRY_ON_STREAM_PARSE_ERROR=false -POLYWEATHER_SCAN_AI_CACHE_TTL_SEC=1800 -POLYWEATHER_SCAN_AI_MAX_ROWS=40 -POLYWEATHER_SCAN_AI_MAX_TOKENS=3200 -POLYWEATHER_SCAN_CITY_AI_MAX_TOKENS=900 -POLYWEATHER_SCAN_AI_PROXY_TIMEOUT_MS=55000 POLYWEATHER_CITY_SUMMARY_CACHE_TTL_SEC=1800 POLYWEATHER_CITY_PANEL_CACHE_TTL_SEC=1800 POLYWEATHER_CITY_NEARBY_CACHE_TTL_SEC=1800 diff --git a/frontend/components/dashboard/scan-terminal/LiveTemperatureThresholdChart.tsx b/frontend/components/dashboard/scan-terminal/LiveTemperatureThresholdChart.tsx index 79589cee..6e1b348f 100644 --- a/frontend/components/dashboard/scan-terminal/LiveTemperatureThresholdChart.tsx +++ b/frontend/components/dashboard/scan-terminal/LiveTemperatureThresholdChart.tsx @@ -22,6 +22,7 @@ import type { DailyModelForecast, } from "@/lib/dashboard-types"; import { buildDebBaselinePath } from "@/lib/temperature-chart-paths"; +import { DASHBOARD_REFRESH_POLICY_MS } from "@/lib/refresh-policy"; import { Panel } from "@/components/dashboard/scan-terminal/Panel"; import { rowName, temp } from "@/components/dashboard/scan-terminal/utils"; @@ -176,7 +177,7 @@ type RunwayHistorySeries = { }; const MAX_OBS_POINTS = 1440; -const HOURLY_CACHE_TTL_MS = 30 * 60 * 1000; +const HOURLY_CACHE_TTL_MS = DASHBOARD_REFRESH_POLICY_MS.observation; const FULL_DAY_SLOT_MINUTES = 30; const FULL_DAY_SLOTS = 48; const SLOT_INTERVAL_MS = FULL_DAY_SLOT_MINUTES * 60 * 1000; diff --git a/frontend/components/dashboard/scan-terminal/MarketOverviewBanner.module.css b/frontend/components/dashboard/scan-terminal/MarketOverviewBanner.module.css deleted file mode 100644 index e9502ba0..00000000 --- a/frontend/components/dashboard/scan-terminal/MarketOverviewBanner.module.css +++ /dev/null @@ -1,178 +0,0 @@ -.root { - display: flex; - flex-direction: column; - border: 1px solid var(--color-border-default, rgba(159, 178, 199, 0.16)); - border-radius: var(--radius-md, 10px); - background: var(--color-bg-card, rgba(17, 26, 46, 0.88)); - backdrop-filter: var(--glass-blur-1, blur(10px)); - margin: 0 0 var(--space-3, 12px) 0; - transition: background 0.2s; -} - -.root.loading { - opacity: 0.7; - flex-direction: row; - align-items: center; - gap: 8px; - padding: 10px 14px; - font-size: 13px; - color: var(--color-text-secondary, #9FB2C7); -} - -.icon { - color: var(--color-accent-primary, #4DA3FF); - flex-shrink: 0; -} - -.header { - display: flex; - align-items: center; - gap: 8px; - padding: 10px 14px; - cursor: pointer; - border: none; - background: transparent; - color: var(--color-text-primary, #E6EDF3); - font-size: 13px; - font-family: inherit; - text-align: left; - width: 100%; -} - -.header:hover { - background: rgba(77, 163, 255, 0.04); -} - -.preview { - flex: 1; - min-width: 0; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - color: var(--color-text-secondary, #9FB2C7); - font-size: 12px; -} - -.toggle { - flex-shrink: 0; - color: var(--color-text-muted, #7D8FA3); -} - -.body { - padding: 0 14px 14px; - display: flex; - flex-direction: column; - gap: 10px; -} - -.summary { - margin: 0; - font-size: 13px; - line-height: 1.65; - color: var(--color-text-primary, #E6EDF3); -} - -.highlights { - list-style: none; - margin: 0; - padding: 0; - display: flex; - flex-wrap: wrap; - gap: 8px; -} - -.highlightItem { - display: inline-flex; - align-items: baseline; - gap: 5px; - padding: 5px 10px; - border-radius: var(--radius-sm, 6px); - background: rgba(77, 163, 255, 0.08); - border: 1px solid rgba(77, 163, 255, 0.14); - font-size: 12px; - line-height: 1.5; - color: var(--color-text-secondary, #9FB2C7); - max-width: 100%; -} - -.highlightItem strong { - color: var(--color-accent-primary, #4DA3FF); - font-weight: 700; - flex-shrink: 0; -} - -.time { - font-size: 11px; - color: var(--color-text-muted, #7D8FA3); -} - -/* Mobile < 768px */ -@media (max-width: 768px) { - .header { - padding: 8px 10px; - font-size: 12px; - } - - .body { - padding: 0 10px 10px; - } - - .summary { - font-size: 12px; - } - - .highlights { - flex-direction: column; - gap: 6px; - } - - .highlightItem { - font-size: 11px; - padding: 4px 8px; - } - -} - -/* Mobile < 640px */ -@media (max-width: 640px) { - .header { - padding: 6px 8px; - } - - .preview { - display: none; - } - - .body { - padding: 0 8px 8px; - } - - .summary { - font-size: 11px; - line-height: 1.55; - } -} - -/* Light theme */ -:global(.scan-terminal.light) .root { - background: rgba(255, 255, 255, 0.78); - border-color: rgba(15, 23, 42, 0.1); -} - -:global(.scan-terminal.light) .summary { - color: #0F172A; -} - -:global(.scan-terminal.light) .preview, -:global(.scan-terminal.light) .highlightItem { - color: #475569; -} - -:global(.scan-terminal.light) .highlightItem { - background: rgba(77, 163, 255, 0.06); - border-color: rgba(77, 163, 255, 0.18); -} - -:global(.scan-terminal.light) .header:hover { - background: rgba(77, 163, 255, 0.05); -} diff --git a/frontend/components/dashboard/scan-terminal/MarketOverviewBanner.tsx b/frontend/components/dashboard/scan-terminal/MarketOverviewBanner.tsx deleted file mode 100644 index a2e3da75..00000000 --- a/frontend/components/dashboard/scan-terminal/MarketOverviewBanner.tsx +++ /dev/null @@ -1,132 +0,0 @@ -"use client"; - -import clsx from "clsx"; -import { ChevronDown, ChevronUp, Sparkles } from "lucide-react"; -import { useCallback, useEffect, useRef, useState } from "react"; -import { fetchBackendApi, buildBrowserBackendHeaders } from "@/lib/backend-api"; -import styles from "./MarketOverviewBanner.module.css"; -import type { ScanOpportunityRow } from "@/lib/dashboard-types"; - -interface OverviewPayload { - overview_zh: string; - overview_en: string; - highlights: Array<{ city: string; note_zh: string; note_en: string }>; - generated_at: string | null; -} - -export function MarketOverviewBanner({ - isEn, - isPro, - rows, -}: { - isEn: boolean; - isPro: boolean; - rows: ScanOpportunityRow[]; -}) { - const [collapsed, setCollapsed] = useState(true); - const [data, setData] = useState(null); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(false); - const fetchedRef = useRef(false); - - const fetchOverview = useCallback(async () => { - if (!isPro || !rows.length || fetchedRef.current) return; - fetchedRef.current = true; - setLoading(true); - setError(false); - try { - const headers = await buildBrowserBackendHeaders({ - "Content-Type": "application/json", - }); - const resp = await fetchBackendApi("/api/scan/terminal/overview", { - method: "POST", - headers, - body: JSON.stringify({ - rows: rows.slice(0, 40).map((row) => ({ - city: row.city ?? row.display_name ?? "", - display_name: row.display_name ?? row.city ?? "", - local_date: row.local_date ?? "", - deb_prediction: row.deb_prediction ?? null, - current_temp: row.current_temp ?? null, - current_max_so_far: row.current_max_so_far ?? null, - risk_level: row.risk_level ?? "", - temp_symbol: row.temp_symbol ?? "°C", - })), - }), - }); - if (resp.ok) { - const json = await resp.json(); - setData(json); - } else { - setError(true); - } - } catch { - setError(true); - } finally { - setLoading(false); - } - }, [isPro, rows]); - - useEffect(() => { - if (isPro && rows.length > 0 && !fetchedRef.current) { - fetchOverview(); - } - }, [isPro, rows.length, fetchOverview]); - - if (!isPro || rows.length === 0) return null; - if (loading && !data) { - return ( -
- - {isEn ? "AI is generating market overview…" : "AI 正在生成市场概览…"} -
- ); - } - if (error && !data) return null; - - const overviewText = data ? (isEn ? data.overview_en : data.overview_zh) : ""; - const highlights = data?.highlights ?? []; - - return ( -
- - - {!collapsed && ( -
-

{overviewText}

- {highlights.length > 0 && ( -
    - {highlights.map((h) => ( -
  • - {h.city} - {isEn ? h.note_en : h.note_zh} -
    • - ))} -
- )} - {data?.generated_at && ( - - )} -
- )} -
- ); -} diff --git a/frontend/components/dashboard/scan-terminal/RealtimeScrollChart.tsx b/frontend/components/dashboard/scan-terminal/RealtimeScrollChart.tsx index 1498fc16..44e3d22b 100644 --- a/frontend/components/dashboard/scan-terminal/RealtimeScrollChart.tsx +++ b/frontend/components/dashboard/scan-terminal/RealtimeScrollChart.tsx @@ -12,6 +12,7 @@ import { YAxis, } from "recharts"; import { Panel } from "@/components/dashboard/scan-terminal/Panel"; +import { DASHBOARD_REFRESH_POLICY_MS } from "@/lib/refresh-policy"; type StreamPoint = { timestamp: string; temp: number; source: string }; type Threshold = { label: string; threshold_c: number; breached: boolean }; @@ -21,7 +22,7 @@ type StreamPayload = { thresholds: Threshold[]; }; -const POLL_INTERVAL_MS = 30_000; +const POLL_INTERVAL_MS = DASHBOARD_REFRESH_POLICY_MS.observation; export function RealtimeScrollChart({ city, diff --git a/frontend/components/dashboard/scan-terminal/__tests__/refreshCadencePolicy.test.ts b/frontend/components/dashboard/scan-terminal/__tests__/refreshCadencePolicy.test.ts new file mode 100644 index 00000000..88a985a9 --- /dev/null +++ b/frontend/components/dashboard/scan-terminal/__tests__/refreshCadencePolicy.test.ts @@ -0,0 +1,48 @@ +import fs from "node:fs"; +import path from "node:path"; +import { + DASHBOARD_REFRESH_POLICY_MS, + DASHBOARD_REFRESH_POLICY_SEC, +} from "@/lib/refresh-policy"; +import { scanTerminalQueryPolicy } from "@/components/dashboard/scan-terminal/scan-terminal-client"; + +function assert(condition: unknown, message: string) { + if (!condition) throw new Error(message); +} + +export function runTests() { + assert(DASHBOARD_REFRESH_POLICY_MS.observation === 60_000, "observation layer should refresh every 60 seconds"); + assert(DASHBOARD_REFRESH_POLICY_MS.scanRows === 5 * 60_000, "region/city rows should refresh every 5 minutes"); + assert(DASHBOARD_REFRESH_POLICY_MS.model === 30 * 60_000, "DEB and multi-model data should refresh every 30 minutes"); + assert(DASHBOARD_REFRESH_POLICY_SEC.metar === 5 * 60, "METAR polling should be 5 minutes"); + assert(scanTerminalQueryPolicy.autoRefreshMs === DASHBOARD_REFRESH_POLICY_MS.scanRows, "scan terminal auto refresh should use the shared row cadence"); + + const projectRoot = process.cwd(); + const querySource = fs.readFileSync( + path.join(projectRoot, "components", "dashboard", "scan-terminal", "use-scan-terminal-query.ts"), + "utf8", + ); + const chartSource = fs.readFileSync( + path.join(projectRoot, "components", "dashboard", "scan-terminal", "LiveTemperatureThresholdChart.tsx"), + "utf8", + ); + const overviewApiSource = fs.readFileSync( + path.join(projectRoot, "..", "web", "services", "market_overview_api.py"), + "utf8", + ); + + assert( + querySource.includes("DASHBOARD_REFRESH_POLICY_MS.scanRows"), + "scan list local cache should use the shared 5-minute row cadence", + ); + assert( + chartSource.includes("DASHBOARD_REFRESH_POLICY_MS.observation"), + "selected city chart detail cache should use the shared 60-second observation cadence", + ); + assert( + !overviewApiSource.includes("/chat/completions") && + !overviewApiSource.includes("SCAN_CITY_AI_MODEL") && + !overviewApiSource.includes("_scan_ai_api_key"), + "market overview must be deterministic and must not call AI providers", + ); +} diff --git a/frontend/components/dashboard/scan-terminal/scan-terminal-client.ts b/frontend/components/dashboard/scan-terminal/scan-terminal-client.ts index ca95fe6e..a6afce2e 100644 --- a/frontend/components/dashboard/scan-terminal/scan-terminal-client.ts +++ b/frontend/components/dashboard/scan-terminal/scan-terminal-client.ts @@ -4,6 +4,7 @@ import { buildBrowserBackendHeaders, fetchBackendApi, } from "@/lib/backend-api"; +import { DASHBOARD_REFRESH_POLICY_MS } from "@/lib/refresh-policy"; import type { CityDetail, MarketScan, @@ -17,7 +18,7 @@ export type RemoteData = | { status: "error"; error: string; previous?: T }; export const scanTerminalQueryPolicy = { - autoRefreshMs: 10 * 60_000, + autoRefreshMs: DASHBOARD_REFRESH_POLICY_MS.scanRows, manualForceRefreshCooldownMs: 2 * 60_000, } as const; diff --git a/frontend/components/dashboard/scan-terminal/use-scan-terminal-query.ts b/frontend/components/dashboard/scan-terminal/use-scan-terminal-query.ts index 71c28925..2d581714 100644 --- a/frontend/components/dashboard/scan-terminal/use-scan-terminal-query.ts +++ b/frontend/components/dashboard/scan-terminal/use-scan-terminal-query.ts @@ -9,10 +9,11 @@ import { } from "@/components/dashboard/scan-terminal/scan-terminal-client"; import { useRemoteDataQuery } from "@/components/dashboard/scan-terminal/use-remote-data-query"; import { REGIONS } from "@/components/dashboard/scan-terminal/continent-grouping"; +import { DASHBOARD_REFRESH_POLICY_MS } from "@/lib/refresh-policy"; import type { ScanTerminalResponse } from "@/lib/dashboard-types"; const SCAN_CACHE_PREFIX = "polyweather_scan_v2"; -const SCAN_CACHE_TTL_MS = 10 * 60 * 1000; // 10 min — cities list instant on revisit +const SCAN_CACHE_TTL_MS = DASHBOARD_REFRESH_POLICY_MS.scanRows; function scanCacheKey(tradingRegion: string): string { return `${SCAN_CACHE_PREFIX}:${tradingRegion || "all"}`; diff --git a/frontend/lib/refresh-policy.ts b/frontend/lib/refresh-policy.ts new file mode 100644 index 00000000..c9cb3582 --- /dev/null +++ b/frontend/lib/refresh-policy.ts @@ -0,0 +1,13 @@ +export const DASHBOARD_REFRESH_POLICY_SEC = { + observation: 60, + metar: 5 * 60, + scanRows: 5 * 60, + model: 30 * 60, +} as const; + +export const DASHBOARD_REFRESH_POLICY_MS = { + observation: DASHBOARD_REFRESH_POLICY_SEC.observation * 1000, + metar: DASHBOARD_REFRESH_POLICY_SEC.metar * 1000, + scanRows: DASHBOARD_REFRESH_POLICY_SEC.scanRows * 1000, + model: DASHBOARD_REFRESH_POLICY_SEC.model * 1000, +} as const; diff --git a/frontend/lib/source-freshness.ts b/frontend/lib/source-freshness.ts index e5df93bd..e693db7e 100644 --- a/frontend/lib/source-freshness.ts +++ b/frontend/lib/source-freshness.ts @@ -1,4 +1,5 @@ import type { CityDetail, ObservationFreshness } from "@/lib/dashboard-types"; +import { DASHBOARD_REFRESH_POLICY_SEC } from "@/lib/refresh-policy"; import { normalizeObservationSourceCode } from "@/lib/source-labels"; export type MonitorFreshnessLevel = "fresh" | "aging" | "stale" | "unknown"; @@ -27,27 +28,27 @@ const DEFAULT_SOURCE_PROFILE: SourceProfile = { freshWindowSec: 600, expectedGraceSec: 900, staleAfterSec: 3600, - pollIntervalSec: 300, + pollIntervalSec: DASHBOARD_REFRESH_POLICY_SEC.metar, }; const SOURCE_PROFILES: Record = { amsc_awos: { code: "amsc_awos", label: "AMSC AWOS", - nativeUpdateIntervalSec: 60, + nativeUpdateIntervalSec: DASHBOARD_REFRESH_POLICY_SEC.observation, freshWindowSec: 180, expectedGraceSec: 180, staleAfterSec: 900, - pollIntervalSec: 60, + pollIntervalSec: DASHBOARD_REFRESH_POLICY_SEC.observation, }, amos: { code: "amos", label: "AMOS", - nativeUpdateIntervalSec: 60, + nativeUpdateIntervalSec: DASHBOARD_REFRESH_POLICY_SEC.observation, freshWindowSec: 180, expectedGraceSec: 180, staleAfterSec: 900, - pollIntervalSec: 60, + pollIntervalSec: DASHBOARD_REFRESH_POLICY_SEC.observation, }, jma: { code: "jma", @@ -79,11 +80,11 @@ const SOURCE_PROFILES: Record = { hko: { code: "hko", label: "HKO", - nativeUpdateIntervalSec: 600, - freshWindowSec: 900, - expectedGraceSec: 600, - staleAfterSec: 2700, - pollIntervalSec: 300, + nativeUpdateIntervalSec: DASHBOARD_REFRESH_POLICY_SEC.observation, + freshWindowSec: 180, + expectedGraceSec: 180, + staleAfterSec: 900, + pollIntervalSec: DASHBOARD_REFRESH_POLICY_SEC.observation, }, cwa: { code: "cwa", diff --git a/src/data_collection/weather_sources.py b/src/data_collection/weather_sources.py index afb9cf31..7c3fbae5 100644 --- a/src/data_collection/weather_sources.py +++ b/src/data_collection/weather_sources.py @@ -7,6 +7,11 @@ from typing import Any, Optional, Dict, List from datetime import datetime, timedelta from loguru import logger from src.data_collection.open_meteo_cache import OpenMeteoCacheMixin +from src.utils.refresh_policy import ( + METAR_POLL_TTL_SEC, + MODEL_CACHE_TTL_SEC, + OBSERVATION_REFRESH_SEC, +) from src.data_collection.settlement_sources import SettlementSourceMixin from src.data_collection.metar_sources import MetarSourceMixin from src.data_collection.mgm_sources import MgmSourceMixin @@ -163,13 +168,13 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour limits=httpx.Limits(max_connections=50, max_keepalive_connections=20), ) self.open_meteo_cache_ttl_sec = int( - os.getenv("OPEN_METEO_CACHE_TTL_SEC", "900") + os.getenv("OPEN_METEO_CACHE_TTL_SEC", str(MODEL_CACHE_TTL_SEC)) ) self.open_meteo_ensemble_cache_ttl_sec = int( - os.getenv("OPEN_METEO_ENSEMBLE_CACHE_TTL_SEC", "900") + os.getenv("OPEN_METEO_ENSEMBLE_CACHE_TTL_SEC", str(MODEL_CACHE_TTL_SEC)) ) self.open_meteo_multi_model_cache_ttl_sec = int( - os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_TTL_SEC", "900") + os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_TTL_SEC", str(MODEL_CACHE_TTL_SEC)) ) self.multi_model_cache_version = str( os.getenv("OPEN_METEO_MULTI_MODEL_CACHE_VERSION", "v3") @@ -193,10 +198,10 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._open_meteo_last_call_ts: float = 0.0 self._open_meteo_call_lock = threading.Lock() self.metar_cache_ttl_sec = int( - os.getenv("METAR_CACHE_TTL_SEC", "600") # 默认 10 分钟 + os.getenv("METAR_CACHE_TTL_SEC", str(METAR_POLL_TTL_SEC)) ) self.metar_fast_cache_ttl_sec = int( - os.getenv("METAR_FAST_CACHE_TTL_SEC", "60") + os.getenv("METAR_FAST_CACHE_TTL_SEC", str(OBSERVATION_REFRESH_SEC)) ) self._metar_cache: Dict[str, Dict] = {} self._metar_cache_lock = threading.Lock() @@ -211,7 +216,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._jma_cache: Dict[str, Dict] = {} self._jma_cache_lock = threading.Lock() self.settlement_cache_ttl_sec = int( - os.getenv("SETTLEMENT_SOURCE_CACHE_TTL_SEC", "120") + os.getenv("SETTLEMENT_SOURCE_CACHE_TTL_SEC", str(OBSERVATION_REFRESH_SEC)) ) self._settlement_cache: Dict[str, Dict] = {} self._settlement_cache_lock = threading.Lock() @@ -226,7 +231,7 @@ class WeatherDataCollector(OpenMeteoCacheMixin, SettlementSourceMixin, MetarSour self._knmi_cache: Dict[str, Dict] = {} self._knmi_cache_lock = threading.Lock() self.hko_obs_cache_ttl_sec = int( - os.getenv("HKO_OBS_CACHE_TTL_SEC", "60") + os.getenv("HKO_OBS_CACHE_TTL_SEC", str(OBSERVATION_REFRESH_SEC)) ) self._hko_obs_cache: Dict[str, Dict] = {} self.madis_cache_ttl_sec = int( diff --git a/src/utils/refresh_policy.py b/src/utils/refresh_policy.py new file mode 100644 index 00000000..504a5e20 --- /dev/null +++ b/src/utils/refresh_policy.py @@ -0,0 +1,9 @@ +"""Shared refresh cadence policy for the web terminal and data collectors.""" + +from __future__ import annotations + +OBSERVATION_REFRESH_SEC = 60 +METAR_POLL_TTL_SEC = 5 * 60 +SCAN_ROWS_REFRESH_SEC = 5 * 60 +MODEL_CACHE_TTL_SEC = 30 * 60 + diff --git a/tests/test_refresh_policy.py b/tests/test_refresh_policy.py new file mode 100644 index 00000000..8a9f72fd --- /dev/null +++ b/tests/test_refresh_policy.py @@ -0,0 +1,35 @@ +from src.utils.refresh_policy import ( + MARKET_OVERVIEW_TTL_SEC, + METAR_POLL_TTL_SEC, + MODEL_CACHE_TTL_SEC, + OBSERVATION_REFRESH_SEC, + SCAN_ROWS_REFRESH_SEC, +) + + +def test_refresh_policy_cadences_are_layered(): + assert OBSERVATION_REFRESH_SEC == 60 + assert METAR_POLL_TTL_SEC == 300 + assert SCAN_ROWS_REFRESH_SEC == 300 + assert MARKET_OVERVIEW_TTL_SEC == 600 + assert MODEL_CACHE_TTL_SEC == 1800 + + +def test_backend_defaults_use_refresh_policy(): + import src.data_collection.weather_sources as weather_sources + import web.services.city_runtime as city_runtime + import web.services.market_overview_api as market_overview_api + import web.services.scan_ai_config as scan_ai_config + + assert scan_ai_config.SCAN_TERMINAL_PAYLOAD_TTL_SEC == SCAN_ROWS_REFRESH_SEC + assert city_runtime.CITY_FULL_CACHE_TTL_SEC == OBSERVATION_REFRESH_SEC + assert city_runtime.CITY_PANEL_CACHE_TTL_SEC == SCAN_ROWS_REFRESH_SEC + assert city_runtime.CITY_MARKET_CACHE_TTL_SEC == SCAN_ROWS_REFRESH_SEC + assert market_overview_api.OVERVIEW_CACHE_TTL_SEC == MARKET_OVERVIEW_TTL_SEC + + source = weather_sources.WeatherDataCollector() + assert source.metar_cache_ttl_sec == METAR_POLL_TTL_SEC + assert source.hko_obs_cache_ttl_sec == OBSERVATION_REFRESH_SEC + assert source.settlement_cache_ttl_sec == OBSERVATION_REFRESH_SEC + assert source.open_meteo_cache_ttl_sec == MODEL_CACHE_TTL_SEC + assert source.open_meteo_multi_model_cache_ttl_sec == MODEL_CACHE_TTL_SEC diff --git a/tests/test_scan_terminal_modules.py b/tests/test_scan_terminal_modules.py index bbf40d21..92fb0b5a 100644 --- a/tests/test_scan_terminal_modules.py +++ b/tests/test_scan_terminal_modules.py @@ -1,4 +1,3 @@ -from web.scan_terminal_ai_merge import merge_scan_ai_result from web.scan_terminal_filters import normalize_scan_terminal_filters from web.scan_terminal_metar_gate import _apply_metar_gate_to_row from web.scan_terminal_payloads import ( @@ -151,55 +150,3 @@ def test_metar_gate_vetoes_yes_when_observed_breaks_above_bucket(): assert "越过目标桶上沿" in row["ai_reason_zh"] -def test_merge_scan_ai_result_applies_city_forecast_and_metadata(): - payload = { - "snapshot_id": "scan-abc", - "rows": [ - { - "id": "row-1", - "city": "manila", - "city_display_name": "Manila", - "final_score": 80.0, - "edge_percent": 5.0, - "metar_context": {"obs_count": 0}, - } - ], - } - ai_raw = { - "summary_zh": "摘要", - "city_forecasts": [ - { - "city": "Manila", - "predicted_max": 35.0, - "range_low": 34.0, - "range_high": 36.0, - "reasoning_zh": "实测突破", - } - ], - "recommendations": [{"row_id": "row-1", "rank": 1, "reason_zh": "观察"}], - "_polyweather_input_meta": {"sent_cities": 1, "sent_contracts": 1}, - } - - merged = merge_scan_ai_result( - payload, - ai_raw, - model="mimo-v2.5-pro", - max_rows=40, - timeout_sec=40, - cache_ttl_sec=1800, - base_url="https://token-plan-cn.xiaomimimo.com/v1", - provider="mimo", - duration_ms=123, - input_rows=1, - ) - - row = merged["rows"][0] - assert row["ai_predicted_max"] == 35.0 - assert row["ai_forecast_reason_zh"] == "实测突破" - assert row["ai_decision"] == "approve" - assert merged["ai_scan"]["sent_cities"] == 1 - assert merged["ai_scan"]["sent_rows"] == 1 - assert merged["ai_scan"]["duration_ms"] == 123 - assert merged["ai_scan"]["model"] == "mimo-v2.5-pro" - assert merged["ai_scan"]["provider"] == "mimo" - assert merged["ai_scan"]["base_url"] == "https://token-plan-cn.xiaomimimo.com/v1" diff --git a/web/analysis_service.py b/web/analysis_service.py index 9712f328..c8227b9d 100644 --- a/web/analysis_service.py +++ b/web/analysis_service.py @@ -297,18 +297,6 @@ def _set_cached_summary(city: str, payload: Dict[str, Any]) -> None: _SUMMARY_CACHE[city] = {"t": _time.time(), "d": dict(payload)} -def _maybe_enrich_dynamic_commentary_with_groq( - _city: str, - result: Dict[str, Any], -) -> Dict[str, Any]: - return result.get("dynamic_commentary") or {"summary": "", "notes": []} - - - - - - - @@ -667,7 +655,6 @@ def _analyze( city: str, force_refresh: bool = False, force_refresh_observations_only: bool = False, - include_llm_commentary: bool = False, detail_mode: str = "full", ) -> Dict[str, Any]: """Fetch, analyse, and return structured weather data for one city. @@ -693,14 +680,6 @@ def _analyze( if not force_refresh and not force_refresh_observations_only: cached = _cache.get(cache_key) if cached and _time.time() - cached["t"] < ttl: - if include_llm_commentary: - cached_payload = cached["d"] - dynamic = cached_payload.get("dynamic_commentary") or {} - if not dynamic.get("headline_zh"): - cached_payload["dynamic_commentary"] = _maybe_enrich_dynamic_commentary_with_groq( - city, - cached_payload, - ) _record_analysis_cache_event(city=city, hit=True, force_refresh=False) return cached["d"] _record_analysis_cache_event(city=city, hit=False, force_refresh=force_refresh) @@ -1714,12 +1693,6 @@ def _analyze( if normalized_detail_mode == "full": _archive_intraday_path_snapshot(city, result) - if include_llm_commentary: - result["dynamic_commentary"] = _maybe_enrich_dynamic_commentary_with_groq( - city, - result, - ) - with _CACHE_LOCK: _cache[cache_key] = {"t": _time.time(), "d": result} return result diff --git a/web/routers/scan.py b/web/routers/scan.py index 23763777..5b77c26d 100644 --- a/web/routers/scan.py +++ b/web/routers/scan.py @@ -5,8 +5,6 @@ from __future__ import annotations from fastapi import APIRouter, Request from web.services.scan_api import ( - get_scan_city_ai_forecast_payload, - get_scan_city_ai_stream_response, get_scan_terminal_ai_payload, get_scan_terminal_overview_payload, get_scan_terminal_payload, @@ -54,16 +52,6 @@ async def scan_terminal_ai(request: Request): return await get_scan_terminal_ai_payload(request) -@router.post("/api/scan/terminal/ai-city") -async def scan_terminal_ai_city(request: Request): - return await get_scan_city_ai_forecast_payload(request) - - -@router.post("/api/scan/terminal/ai-city/stream") -async def scan_terminal_ai_city_stream(request: Request): - return await get_scan_city_ai_stream_response(request) - - @router.post("/api/scan/terminal/overview") async def scan_terminal_overview(request: Request): return await get_scan_terminal_overview_payload(request) diff --git a/web/scan_city_ai_fallback.py b/web/scan_city_ai_fallback.py deleted file mode 100644 index 93abab7c..00000000 --- a/web/scan_city_ai_fallback.py +++ /dev/null @@ -1,507 +0,0 @@ -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from web.scan_city_ai_helpers import ( - CITY_AI_REQUIRED_FIELDS, - _CITY_AI_TEXT_FIELDS, - _extract_city_ai_partial_fields, - _provider_response_meta, - _safe_float, - _strip_incomplete_ai_sentence, - _truncate_ai_text, -) - - -def _format_ai_temperature(value: Any, unit: str) -> Optional[str]: - numeric = _safe_float(value) - if numeric is None: - return None - return f"{numeric:.1f}{unit or ''}" - - -def _city_ai_model_cluster_note(ai_input: Dict[str, Any], *, locale: str) -> str: - observation_anchor = ai_input.get("observation_anchor") if isinstance(ai_input.get("observation_anchor"), dict) else {} - is_hko_observation = observation_anchor.get("is_airport_metar") is False - observation_label = ( - "HKO observations" - if locale == "en-US" and is_hko_observation - else "香港天文台观测" - if is_hko_observation - else "METAR" - ) - cluster = ai_input.get("model_cluster") if isinstance(ai_input.get("model_cluster"), dict) else {} - sources = cluster.get("sources") if isinstance(cluster.get("sources"), list) else [] - unit = str(ai_input.get("temp_symbol") or "") - values = [ - _safe_float(item.get("value")) - for item in sources - if isinstance(item, dict) and _safe_float(item.get("value")) is not None - ] - count = len(values) - deb_value = next( - (_safe_float(item.get("value")) for item in sources if isinstance(item, dict) and "DEB" in str(item.get("model") or "")), - None, - ) - if locale == "en-US": - if count <= 0: - return f"No usable model cluster was returned; rely on {observation_label} only." - if count <= 2: - return f"Only {count} model source(s) are available, so model support is thin and should be treated as context." - range_text = f"{min(values):.1f}{unit} to {max(values):.1f}{unit}" - if deb_value is None: - return f"{count} model sources cluster between {range_text}." - return f"{count} model sources cluster between {range_text}; DEB sits at {deb_value:.1f}{unit}." - if count <= 0: - return f"没有可用的多模型集合，只能把{observation_label}作为主要依据。" - if count <= 2: - return f"当前只有 {count} 个模型来源，模型支撑偏薄，只能作为辅助上下文。" - range_text = f"{min(values):.1f}{unit} ~ {max(values):.1f}{unit}" - if deb_value is None: - return f"{count} 个模型集中在 {range_text}。" - return f"{count} 个模型集中在 {range_text}；DEB 位于 {deb_value:.1f}{unit}。" - - -def _build_city_ai_fallback( - ai_input: Dict[str, Any], - *, - locale: str, - reason: str, - raw_content: str = "", - provider_data: Any = None, -) -> Dict[str, Any]: - unit = str(ai_input.get("temp_symbol") or "") - cluster = ai_input.get("model_cluster") if isinstance(ai_input.get("model_cluster"), dict) else {} - all_sources = cluster.get("sources") if isinstance(cluster.get("sources"), list) else [] - values = [ - _safe_float(item.get("value")) - for item in all_sources - if isinstance(item, dict) and _safe_float(item.get("value")) is not None - ] - deb_value = next( - (_safe_float(item.get("value")) for item in all_sources if isinstance(item, dict) and "DEB" in str(item.get("model") or "")), - None, - ) - non_deb_values = [ - _safe_float(item.get("value")) - for item in all_sources - if isinstance(item, dict) and _safe_float(item.get("value")) is not None and "DEB" not in str(item.get("model") or "") - ] - cluster_median = ( - sorted(non_deb_values)[len(non_deb_values) // 2] - if non_deb_values - else (sorted(values)[len(values) // 2] if values else None) - ) - observation_anchor = ai_input.get("observation_anchor") if isinstance(ai_input.get("observation_anchor"), dict) else {} - is_airport_metar = observation_anchor.get("is_airport_metar") is not False - airport_current = ai_input.get("airport_current") if isinstance(ai_input.get("airport_current"), dict) else {} - current_obs = ai_input.get("current") if isinstance(ai_input.get("current"), dict) else {} - metar_context = ai_input.get("metar_context") if isinstance(ai_input.get("metar_context"), dict) else {} - observation_stale = bool( - metar_context.get("stale_for_today") - or airport_current.get("stale_for_today") - or current_obs.get("stale_for_today") - ) - current_temp = _safe_float( - airport_current.get("temp") if is_airport_metar else current_obs.get("temp") - ) - current_max_so_far = _safe_float( - ai_input.get("current_max_so_far") - or current_obs.get("max_so_far") - or airport_current.get("max_so_far") - ) - observed_high_so_far = max( - [value for value in (current_temp, current_max_so_far) if value is not None], - default=None, - ) - observed_high_for_revision = None if observation_stale else observed_high_so_far - predicted = cluster_median - if predicted is None and deb_value is not None: - predicted = deb_value - if predicted is None and values: - predicted = sum(values) / len(values) - if predicted is None: - predicted = observed_high_so_far - range_low = min(values) if values else predicted - range_high = max(values) if values else predicted - peak = ai_input.get("peak") if isinstance(ai_input.get("peak"), dict) else {} - window_phase = str( - ai_input.get("window_phase") - or peak.get("window_phase") - or peak.get("phase") - or "" - ).strip().lower() - remaining_window_minutes = _safe_float( - ai_input.get("remaining_window_minutes") - if ai_input.get("remaining_window_minutes") is not None - else peak.get("remaining_window_minutes") - if peak.get("remaining_window_minutes") is not None - else peak.get("remaining_minutes") - ) - minutes_until_peak_start = _safe_float( - ai_input.get("minutes_until_peak_start") - if ai_input.get("minutes_until_peak_start") is not None - else peak.get("minutes_until_peak_start") - ) - minutes_until_peak_end = _safe_float( - ai_input.get("minutes_until_peak_end") - if ai_input.get("minutes_until_peak_end") is not None - else peak.get("minutes_until_peak_end") - ) - peak_window_label = str( - ai_input.get("peak_window_label") - or peak.get("peak_window_label") - or peak.get("label") - or "" - ).strip() - peak_has_passed = ( - window_phase in {"post_peak", "past"} - or (minutes_until_peak_end is not None and minutes_until_peak_end < 0) - ) - peak_is_closing = ( - window_phase == "active_peak" - and remaining_window_minutes is not None - and remaining_window_minutes <= 90 - ) - peak_not_started = ( - window_phase in {"early_today", "setup_today", "tomorrow", "week_ahead"} - or (minutes_until_peak_start is not None and minutes_until_peak_start > 0) - ) - model_range_high = range_high - model_range_low = range_low - current_above_predicted = ( - observed_high_for_revision is not None - and predicted is not None - and observed_high_for_revision > predicted + 0.2 - ) - current_above_model_range = ( - observed_high_for_revision is not None - and model_range_high is not None - and observed_high_for_revision > model_range_high + 0.2 - ) - observed_high_break = bool(current_above_predicted or current_above_model_range) - current_below_predicted = ( - observed_high_for_revision is not None - and predicted is not None - and observed_high_for_revision < predicted - 1.5 - ) - current_below_model_range = ( - observed_high_for_revision is not None - and model_range_low is not None - and observed_high_for_revision < model_range_low - 0.2 - ) - observed_low_break = bool(current_below_predicted and (peak_has_passed or peak_is_closing)) - observed_low_lag = bool(current_below_predicted and not observed_low_break) - original_predicted = predicted - if observed_high_break: - predicted = max( - value - for value in (predicted, observed_high_for_revision) - if value is not None - ) - if range_high is not None and observed_high_for_revision is not None: - range_high = max(range_high, observed_high_for_revision) - elif observed_low_break: - predicted = min( - value - for value in (predicted, observed_high_for_revision) - if value is not None - ) - if range_low is not None and observed_high_for_revision is not None: - range_low = min(range_low, observed_high_for_revision) - city = str(ai_input.get("city_display_name") or ai_input.get("city") or "this city") - station = str((airport_current.get("station_code") if is_airport_metar else None) or observation_anchor.get("station_code") or current_obs.get("station_code") or "") - raw_metar = str(airport_current.get("raw_metar") or "").strip() if is_airport_metar else "" - metar_temp = _format_ai_temperature((airport_current.get("temp") if is_airport_metar else current_obs.get("temp")), unit) - obs_time = str((airport_current.get("report_time") or airport_current.get("obs_time")) if is_airport_metar else (current_obs.get("obs_time") or current_obs.get("report_time") or "")).strip() - source_name_zh = "METAR" if is_airport_metar else "香港天文台观测" - source_name_en = "METAR" if is_airport_metar else "Hong Kong Observatory observation" - bulletin_zh = "机场报文" if is_airport_metar else "官方观测" - bulletin_en = "airport-bulletin" if is_airport_metar else "official-station observation" - model_note_zh = _city_ai_model_cluster_note(ai_input, locale="zh-CN") - model_note_en = _city_ai_model_cluster_note(ai_input, locale="en-US") - content_preview = _truncate_ai_text(raw_content, 1000) - partial_ai = _extract_city_ai_partial_fields(raw_content) - looks_like_truncated_json = bool(content_preview.startswith("{") and not content_preview.rstrip().endswith("}")) - reason_preview = _truncate_ai_text(reason, 260) - reason_lower = str(reason or "").lower() - timed_out = "timeout" in reason_lower or "timed out" in reason_lower or "超时" in str(reason or "") - if partial_ai.get("metar_read_zh") or partial_ai.get("metar_read_en"): - metar_zh = str(partial_ai.get("metar_read_zh") or partial_ai.get("metar_read_en") or "").strip() - metar_en = str(partial_ai.get("metar_read_en") or partial_ai.get("metar_read_zh") or "").strip() - elif content_preview and not looks_like_truncated_json: - metar_zh = f"{bulletin_zh}快速解读已先完成；AI 补充摘要：{content_preview}" - metar_en = f"The fast {bulletin_en} read is available; AI supplemental summary: {content_preview}" - elif content_preview: - metar_zh = f"{bulletin_zh}快速解读已先完成；本轮 AI 增强未完整返回，当前以 DEB、多模型与{source_name_zh}为准。" - metar_en = f"The fast {bulletin_en} read is available; this AI enhancement was incomplete, so DEB, model cluster and {source_name_en} carry the read." - elif raw_metar and observation_stale: - metar_zh = f"{station} 可用 METAR 显示 {metar_temp or '温度未知'}，报文时间 {obs_time or '未知'}；但该观测已标记为过旧，当前只能作为背景参考，不能作为强实况锚点。" - metar_en = f"{station} available METAR shows {metar_temp or 'unknown temperature'} at {obs_time or 'unknown time'}, but the observation is flagged as stale, so treat it as background context rather than a strong live anchor." - elif raw_metar: - metar_zh = f"{station} 最新 METAR 显示 {metar_temp or '温度未知'}，报文时间 {obs_time or '未知'}；当前先把它作为实况锚点，并结合后续报文确认温度路径。" - metar_en = f"{station} latest METAR shows {metar_temp or 'unknown temperature'} at {obs_time or 'unknown time'}; use it as the live anchor while later reports confirm the path." - else: - metar_zh = f"当前没有可用的{source_name_zh}正文，暂以 DEB、多模型路径与最新实测为主。" - metar_en = f"No raw {source_name_en} text is available, so DEB, latest observations and the model cluster carry the read." - predicted_text = _format_ai_temperature(predicted, unit) or "--" - current_text = _format_ai_temperature(observed_high_so_far, unit) or "--" - original_predicted_text = _format_ai_temperature(original_predicted, unit) or "--" - model_range_high_text = _format_ai_temperature(model_range_high, unit) or "--" - model_range_low_text = _format_ai_temperature(model_range_low, unit) or "--" - peak_label_text_zh = f"（{peak_window_label}）" if peak_window_label else "" - peak_label_text_en = f" ({peak_window_label})" if peak_window_label else "" - if partial_ai.get("final_judgment_zh") or partial_ai.get("final_judgment_en"): - final_zh = str(partial_ai.get("final_judgment_zh") or partial_ai.get("final_judgment_en") or "").strip() - final_en = str(partial_ai.get("final_judgment_en") or partial_ai.get("final_judgment_zh") or "").strip() - elif partial_ai: - final_zh = f"{city} 预计最高温暂以 {predicted_text} 附近为中枢；AI 已先完成{bulletin_zh}解读，最高温结论结合 DEB、多模型与最新实测校准。" - final_en = f"{city} daily high is centered near {predicted_text}; AI has already read the {bulletin_en}, with the high calibrated against DEB, the model cluster and latest observations." - elif observed_high_break: - final_zh = f"{city} 最新实测已达 {current_text}，高于原先 {original_predicted_text} 中枢；最高温中枢需先上修到至少 {predicted_text} 附近。" - final_en = f"{city} latest observation has reached {current_text}, above the prior {original_predicted_text} center; the daily-high center should be revised up to at least near {predicted_text}." - elif observed_low_break: - final_zh = f"{city} 峰值窗口{peak_label_text_zh}已过或接近结束，实测最高仍约 {current_text}，低于原先 {original_predicted_text} 中枢；最高温中枢需先下修到 {predicted_text} 附近。" - final_en = f"{city} peak window{peak_label_text_en} has passed or is nearly over, and observed high is still near {current_text}, below the prior {original_predicted_text} center; revise the daily-high center down toward {predicted_text}." - elif peak_has_passed: - final_zh = f"{city} 峰值窗口{peak_label_text_zh}已过；最高温暂以 {predicted_text} 附近为中枢，并以已观测到的高点为主要校准。" - final_en = f"{city} peak window{peak_label_text_en} has passed; the daily high stays centered near {predicted_text}, calibrated mainly against the observed high so far." - elif observation_stale: - final_zh = f"{city} 最高温暂以 {predicted_text} 附近为中枢；当前可用{source_name_zh}已过旧，先以 DEB 和多模型路径为主。" - final_en = f"{city} daily high is centered near {predicted_text}; the available {source_name_en} is stale, so DEB and the model path carry the read for now." - elif timed_out: - final_zh = f"{city} 预计最高温暂以 {predicted_text} 附近为中枢；当前已先用 DEB、多模型和{source_name_zh}快速证据模式判断。" - final_en = f"{city} daily high is centered near {predicted_text}; the current read uses the fast DEB/model/{source_name_en} evidence mode." - else: - final_zh = f"{city} 预计最高温暂以 {predicted_text} 附近为中枢；当前已先用 DEB、多模型和{source_name_zh}快速证据模式判断。" - final_en = f"{city} daily high is centered near {predicted_text}; the current read uses the fast DEB/model/{source_name_en} evidence mode." - if partial_ai: - fallback_reasoning_zh = f"AI {bulletin_zh}解读已用于校准日内节奏；DEB 与多模型集合继续约束最高温中枢，后续{source_name_zh}用于确认是否需要上调或下修。" - fallback_reasoning_en = f"The AI {bulletin_en} read is already used to calibrate the intraday pace; DEB and the model cluster still constrain the high-temperature center, while later {source_name_en} updates confirm whether to revise it." - elif observed_high_break: - fallback_reasoning_zh = f"当前为快速证据模式；最新{source_name_zh}已高于原先 {original_predicted_text} 中枢{('，并超过模型上沿 ' + model_range_high_text) if current_above_model_range else ''}，本轮最高温判断应优先承认实测突破并等待完整 AI {bulletin_zh}解读合并。" - fallback_reasoning_en = f"This is the fast evidence mode; latest {source_name_en} is above the prior {original_predicted_text} center{(' and above the model upper edge ' + model_range_high_text) if current_above_model_range else ''}, so the high-temperature read should first acknowledge the observed break and merge the full AI {bulletin_en} read when available." - elif observed_low_break: - fallback_reasoning_zh = f"当前为快速证据模式；峰值窗口已过或接近结束，最新实测高点仍低于原先 {original_predicted_text} 中枢{('，并低于模型下沿 ' + model_range_low_text) if current_below_model_range else ''}，本轮最高温判断应优先承认下修压力并等待完整 AI {bulletin_zh}解读合并。" - fallback_reasoning_en = f"This is the fast evidence mode; the peak window has passed or is nearly over, and the observed high remains below the prior {original_predicted_text} center{(' and below the model lower edge ' + model_range_low_text) if current_below_model_range else ''}, so the high-temperature read should first acknowledge downward revision pressure and merge the full AI {bulletin_en} read when available." - elif observed_low_lag and peak_not_started: - fallback_reasoning_zh = f"当前为快速证据模式；最新{source_name_zh}仍低于原先 {original_predicted_text} 中枢，但峰值窗口尚未到来，暂不直接下修，只把后续升温是否追上模型路径作为关键确认。" - fallback_reasoning_en = f"This is the fast evidence mode; latest {source_name_en} remains below the prior {original_predicted_text} center, but the peak window has not arrived, so do not revise down yet and use later warming as the key confirmation." - elif peak_has_passed: - fallback_reasoning_zh = f"当前为快速证据模式；峰值窗口已过，后续{source_name_zh}主要用于确认是否已形成日内高点，而不是继续按待升温路径解读。" - fallback_reasoning_en = f"This is the fast evidence mode; the peak window has passed, so later {source_name_en} updates mainly confirm whether the daily high is already set rather than assuming further warming." - elif observation_stale: - fallback_reasoning_zh = f"当前为快速证据模式；可用{source_name_zh}已过旧，不能作为强实况锚点，暂由 DEB 和多模型集合支撑本轮最高温中枢，等待新的{source_name_zh}确认。" - fallback_reasoning_en = f"This is the fast evidence mode; the available {source_name_en} is stale and should not be used as a strong live anchor, so DEB and the model cluster carry the current daily-high center until a newer {source_name_en} confirms it." - else: - fallback_reasoning_zh = f"当前为快速证据模式；DEB、多模型集合和最新{source_name_zh}共同支撑本轮最高温中枢，完整 AI {bulletin_zh}解读返回后再合并。" - fallback_reasoning_en = f"This is the fast evidence mode; DEB, the model cluster and latest {source_name_en} jointly support the current daily-high center, and the full AI {bulletin_en} read will be merged when available." - reasoning_zh = str(partial_ai.get("reasoning_zh") or "").strip() or fallback_reasoning_zh - reasoning_en = str(partial_ai.get("reasoning_en") or "").strip() or fallback_reasoning_en - if observed_high_break: - risks_zh = [f"最新{source_name_zh}已突破原模型路径，若后续报文继续持平或升温，需要继续上修最高温中枢。"] - risks_en = [f"Latest {source_name_en} has already broken above the prior model path; if later reports hold steady or warm further, keep revising the daily-high center upward."] - elif observed_low_break: - risks_zh = [f"峰值窗口已过或接近结束且实测仍偏低，若后续{source_name_zh}没有反弹，需要继续下修最高温中枢。"] - risks_en = [f"The peak window has passed or is nearly over while observations remain low; if later {source_name_en} does not rebound, keep revising the daily-high center lower."] - elif observed_low_lag: - risks_zh = [f"最新{source_name_zh}仍未追上原模型路径；若峰值窗口前继续偏低，需要下修最高温中枢。"] - risks_en = [f"Latest {source_name_en} has not caught up with the prior model path; if it stays low before the peak window, revise the daily-high center lower."] - elif peak_has_passed: - risks_zh = [f"峰值窗口已过，后续{source_name_zh}若未再创新高，应避免继续上调最高温中枢。"] - risks_en = [f"The peak window has passed; avoid raising the daily-high center unless later {source_name_en} sets a new high."] - elif observation_stale: - risks_zh = [f"当前{source_name_zh}过旧；新报文若明显偏离 DEB 和模型路径，需要重新校准最高温中枢。"] - risks_en = [f"The current {source_name_en} is stale; if a newer report diverges from DEB and the model path, recalibrate the daily-high center."] - else: - risks_zh = [f"后续{source_name_zh}若明显偏离模型路径，需及时修正最高温中枢。"] - risks_en = [f"If later {source_name_en} updates diverge from the model path, revise the daily-high center promptly."] - evidence_guard = { - "observation_stale": observation_stale, - "observed_high_break": observed_high_break, - "observed_low_break": observed_low_break, - "observed_low_lag": observed_low_lag, - "peak_has_passed": peak_has_passed, - "peak_is_closing": peak_is_closing, - "peak_not_started": peak_not_started, - "observed_high_so_far": observed_high_so_far, - "observed_high_for_revision": observed_high_for_revision, - "original_predicted": original_predicted, - "model_range_low": model_range_low, - "model_range_high": model_range_high, - "predicted_max": predicted, - "range_low": range_low, - "range_high": range_high, - } - taf_data = ai_input.get("taf") if isinstance(ai_input.get("taf"), dict) else {} - if partial_ai.get("taf_read_zh") or partial_ai.get("taf_read_en"): - taf_zh = str(partial_ai.get("taf_read_zh") or partial_ai.get("taf_read_en") or "").strip() - taf_en = str(partial_ai.get("taf_read_en") or partial_ai.get("taf_read_zh") or "").strip() - elif taf_data.get("raw_taf"): - taf_zh = f"TAF 可用，需人工判读：{str(taf_data.get('raw_taf', ''))[:120]}" - taf_en = f"TAF available, manual read: {str(taf_data.get('raw_taf', ''))[:120]}" - else: - taf_zh = "无可用 TAF" - taf_en = "No TAF available" - - prob_data = ai_input.get("probability") if isinstance(ai_input.get("probability"), dict) else {} - if partial_ai.get("probability_read_zh") or partial_ai.get("probability_read_en"): - prob_zh = str(partial_ai.get("probability_read_zh") or partial_ai.get("probability_read_en") or "").strip() - prob_en = str(partial_ai.get("probability_read_en") or partial_ai.get("probability_read_zh") or "").strip() - elif prob_data.get("top_buckets"): - top = prob_data["top_buckets"][0] if isinstance(prob_data["top_buckets"], list) else {} - if isinstance(top, dict) and top.get("label"): - skew_text = f"，分布{'偏右' if prob_data.get('skew') == 'right' else '偏左' if prob_data.get('skew') == 'left' else '对称'}" if prob_data.get("skew") else "" - prob_zh = f"最高概率桶 {top.get('label', '')}（{top.get('prob', '?')}%）{skew_text}" - prob_en = f"Peak bucket {top.get('label', '')} ({top.get('prob', '?')}%){skew_text.replace('偏右', ', right-skewed').replace('偏左', ', left-skewed').replace('对称', ', symmetric')}" - else: - prob_zh = "概率分布数据可用但格式异常" - prob_en = "Probability data available but malformed" - else: - prob_zh = "概率分布暂未生成" - prob_en = "Probability distribution not yet generated" - - return { - "predicted_max": partial_ai.get("predicted_max", predicted), - "range_low": partial_ai.get("range_low", range_low), - "range_high": partial_ai.get("range_high", range_high), - "unit": partial_ai.get("unit") or unit, - "confidence": partial_ai.get("confidence") or ("medium" if partial_ai else "low"), - "final_judgment_zh": final_zh, - "final_judgment_en": final_en, - "metar_read_zh": metar_zh, - "metar_read_en": metar_en, - "taf_read_zh": taf_zh, - "taf_read_en": taf_en, - "probability_read_zh": prob_zh, - "probability_read_en": prob_en, - "reasoning_zh": reasoning_zh, - "reasoning_en": reasoning_en, - "risks_zh": risks_zh, - "risks_en": risks_en, - "model_cluster_note_zh": partial_ai.get("model_cluster_note_zh") or model_note_zh, - "model_cluster_note_en": partial_ai.get("model_cluster_note_en") or model_note_en, - "_polyweather_meta": { - **_provider_response_meta(provider_data), - "fallback": True, - "fallback_kind": "partial_ai_json" if partial_ai else "timeout" if timed_out else "non_json", - "looks_like_truncated_json": looks_like_truncated_json, - "fallback_reason": reason_preview, - "raw_content_preview": content_preview, - "partial_ai_fields": sorted(partial_ai.keys()), - "raw_metar": _truncate_ai_text(raw_metar, 1000), - "evidence_guard": evidence_guard, - }, - } - -def _complete_city_ai_payload( - ai_raw: Dict[str, Any], - ai_input: Dict[str, Any], - *, - locale: str, -) -> Dict[str, Any]: - """Fill missing structured fields without marking a provider success as fallback.""" - if not isinstance(ai_raw, dict): - return _build_city_ai_fallback( - ai_input, - locale=locale, - reason="provider output was not a JSON object", - ) - fallback = _build_city_ai_fallback( - ai_input, - locale=locale, - reason="schema completion", - ) - completed: List[str] = [] - out = dict(ai_raw) - trimmed: List[str] = [] - for field in CITY_AI_REQUIRED_FIELDS: - value = out.get(field) - if isinstance(value, str) and field in _CITY_AI_TEXT_FIELDS: - clean_value = _strip_incomplete_ai_sentence(value) - if clean_value != value: - trimmed.append(field) - value = clean_value - out[field] = clean_value - if value is None or value == "" or value == []: - out[field] = fallback.get(field) - completed.append(field) - for field in ("risks_zh", "risks_en"): - value = out.get(field) - if isinstance(value, list): - cleaned_risks = [] - for item in value: - clean_item = _strip_incomplete_ai_sentence(item) - if clean_item: - cleaned_risks.append(clean_item) - if cleaned_risks != value: - trimmed.append(field) - out[field] = cleaned_risks or fallback.get(field) - meta = out.get("_polyweather_meta") - if not isinstance(meta, dict): - meta = {} - if completed: - meta["schema_completed_fields"] = completed - if trimmed: - meta["trimmed_incomplete_fields"] = sorted(set(trimmed)) - guard_meta = (fallback.get("_polyweather_meta") or {}).get("evidence_guard") - if isinstance(guard_meta, dict): - deterministic_fields: List[str] = [] - numeric_guard_active = bool( - guard_meta.get("observed_high_break") - or guard_meta.get("observed_low_break") - or guard_meta.get("observation_stale") - ) - text_guard_active = bool( - numeric_guard_active - or guard_meta.get("peak_has_passed") - or (guard_meta.get("observed_low_lag") and guard_meta.get("peak_not_started")) - ) - if numeric_guard_active: - for field in ("predicted_max", "range_low", "range_high"): - guarded_value = fallback.get(field) - if guarded_value is not None and out.get(field) != guarded_value: - out[field] = guarded_value - deterministic_fields.append(field) - if guard_meta.get("observation_stale"): - guarded_text_fields = ( - "metar_read_zh", - "metar_read_en", - "final_judgment_zh", - "final_judgment_en", - "reasoning_zh", - "reasoning_en", - "risks_zh", - "risks_en", - ) - elif text_guard_active: - guarded_text_fields = ( - "final_judgment_zh", - "final_judgment_en", - "reasoning_zh", - "reasoning_en", - "risks_zh", - "risks_en", - ) - else: - guarded_text_fields = () - for field in guarded_text_fields: - guarded_value = fallback.get(field) - if guarded_value not in (None, "", []) and out.get(field) != guarded_value: - out[field] = guarded_value - deterministic_fields.append(field) - if deterministic_fields: - meta["deterministic_guard_fields"] = sorted(set(deterministic_fields)) - meta["deterministic_guard_reason"] = { - key: guard_meta.get(key) - for key in ( - "observation_stale", - "observed_high_break", - "observed_low_break", - "observed_low_lag", - "peak_has_passed", - ) - if guard_meta.get(key) - } - out["_polyweather_meta"] = meta - return out diff --git a/web/scan_city_ai_helpers.py b/web/scan_city_ai_helpers.py deleted file mode 100644 index bfc44c97..00000000 --- a/web/scan_city_ai_helpers.py +++ /dev/null @@ -1,322 +0,0 @@ -from __future__ import annotations - -import json -import re -from typing import Any, Dict, List, Optional - -CITY_AI_REQUIRED_FIELDS = [ - "metar_read_zh", - "metar_read_en", - "taf_read_zh", - "taf_read_en", - "probability_read_zh", - "probability_read_en", - "final_judgment_zh", - "final_judgment_en", - "predicted_max", - "range_low", - "range_high", - "unit", - "confidence", - "reasoning_zh", - "reasoning_en", - "risks_zh", - "risks_en", - "model_cluster_note_zh", - "model_cluster_note_en", -] - -CITY_AI_STREAM_PROVIDER_FIELDS = [ - "metar_read_zh", - "metar_read_en", - "taf_read_zh", - "taf_read_en", - "probability_read_zh", - "probability_read_en", - "predicted_max", - "range_low", - "range_high", - "unit", - "confidence", - "final_judgment_zh", - "final_judgment_en", - "reasoning_zh", - "reasoning_en", -] - - - -def _safe_float(value: Any) -> Optional[float]: - try: - if value is None or value == "": - return None - return float(value) - except Exception: - return None - - - -def _extract_ai_json_object(raw_text: str) -> Dict[str, Any]: - text = str(raw_text or "").strip() - if not text: - raise ValueError("empty AI content") - try: - parsed = json.loads(text) - if isinstance(parsed, dict): - return parsed - except Exception: - pass - start = text.find("{") - end = text.rfind("}") - if start >= 0 and end > start: - parsed = json.loads(text[start : end + 1]) - if isinstance(parsed, dict): - return parsed - raise ValueError("AI content is not a JSON object") - - -def _decode_json_string_fragment(fragment: str) -> str: - safe = str(fragment or "") - while safe.endswith("\\"): - safe = safe[:-1] - try: - return str(json.loads(f'"{safe}"')) - except Exception: - return ( - safe.replace('\\"', '"') - .replace("\\n", "\n") - .replace("\\r", "\r") - .replace("\\t", "\t") - .replace("\\\\", "\\") - ) - - -def _extract_json_string_field_fragment(raw_text: str, field: str) -> tuple[str, bool]: - """Best-effort extraction from a streamed/incomplete JSON object. - - DeepSeek may stream useful fields first but still end with truncated JSON. - In that case the UI should keep the AI text already received instead of - replacing it with the deterministic DEB/METAR fallback. - """ - - text = str(raw_text or "") - match = re.search(rf'"{re.escape(field)}"\s*:\s*"', text) - if not match: - return "", False - idx = match.end() - chars: List[str] = [] - escaped = False - closed = False - while idx < len(text): - char = text[idx] - idx += 1 - if escaped: - chars.append("\\" + char) - escaped = False - continue - if char == "\\": - escaped = True - continue - if char == '"': - closed = True - break - chars.append(char) - if escaped: - chars.append("\\") - return _decode_json_string_fragment("".join(chars)).strip(), closed - - -_CITY_AI_TEXT_FIELDS = { - "metar_read_zh", - "metar_read_en", - "taf_read_zh", - "taf_read_en", - "probability_read_zh", - "probability_read_en", - "final_judgment_zh", - "final_judgment_en", - "reasoning_zh", - "reasoning_en", - "model_cluster_note_zh", - "model_cluster_note_en", -} - - -_INCOMPLETE_TAIL_RE = re.compile( - r"(?:(?:[，,；;]\s*)?(?:但|但是|不过|然而|而且|并且|因为|由于|若|如果|but|however|although|because|if|while|and)\s*)?" - r"(?:TAF|METAR|报文|机场预报|机场报文)?\s*(?:显示|提示|预示|表明|show(?:s)?|indicate(?:s)?|suggest(?:s)?)?\s*$", - re.IGNORECASE, -) - - -def _strip_incomplete_ai_sentence(value: Any) -> str: - """Remove dangling provider fragments such as "但TAF显示". - - The city AI endpoint can fall back to partially streamed JSON. If the stream - stops in the middle of a string, keeping the whole fragment produces broken - UI text. Prefer the last complete sentence/clause; if the whole field is too - incomplete, let schema completion use the deterministic fallback. - """ - - text = re.sub(r"\s+", " ", str(value or "")).strip() - if not text: - return "" - if re.search(r"[。！？.!?]$", text): - return text - - stripped = _INCOMPLETE_TAIL_RE.sub("", text).rstrip(" ，,；;：:") - if stripped != text: - text = stripped.strip() - if not text: - return "" - if re.search(r"[。！？.!?]$", text): - return text - return text.rstrip(" ，,；;：:") + ("." if re.search(r"[A-Za-z]$", text) else "。") - - # If the provider stopped after a semicolon/comma-delimited clause, keep the - # complete preceding clause instead of showing a dangling tail. - for punct in ("；", ";", "。", "！", "？", ".", "!", "?"): - idx = text.rfind(punct) - if idx >= 0: - candidate = text[: idx + 1].strip() - if len(candidate) >= 8: - return candidate.rstrip("；;") + ("." if punct == ";" else "。" if punct == "；" else "") - - # If no dangling connector is visible, keep the provider text and add only - # terminal punctuation. This avoids replacing otherwise useful long reads - # just because the JSON string missed its closing quote. - return text.rstrip(" ，,；;：:") + ("." if re.search(r"[A-Za-z]$", text) else "。") - - -def _extract_json_number_field_from_fragment(raw_text: str, field: str) -> Optional[float]: - text = str(raw_text or "") - match = re.search(rf'"{re.escape(field)}"\s*:\s*(-?\d+(?:\.\d+)?)', text) - if not match: - return None - return _safe_float(match.group(1)) - - -def _extract_city_ai_partial_fields(raw_text: str) -> Dict[str, Any]: - text = str(raw_text or "") - if not text.strip(): - return {} - out: Dict[str, Any] = {} - for field in ( - "metar_read_zh", - "metar_read_en", - "final_judgment_zh", - "final_judgment_en", - "reasoning_zh", - "reasoning_en", - "model_cluster_note_zh", - "model_cluster_note_en", - "confidence", - "unit", - ): - value, closed = _extract_json_string_field_fragment(text, field) - if value and field in _CITY_AI_TEXT_FIELDS and not closed: - value = _strip_incomplete_ai_sentence(value) - if value: - out[field] = value - for field in ("predicted_max", "range_low", "range_high"): - value = _extract_json_number_field_from_fragment(text, field) - if value is not None: - out[field] = value - return out - - -def _truncate_ai_text(value: Any, limit: int = 800) -> str: - text = re.sub(r"\s+", " ", str(value or "")).strip() - if len(text) <= limit: - return text - return text[: max(0, limit - 1)].rstrip() + "…" - - -def _extract_provider_content(data: Any) -> str: - if not isinstance(data, dict): - return "" - choices = data.get("choices") or [] - if not choices or not isinstance(choices[0], dict): - return "" - message = choices[0].get("message") or {} - if not isinstance(message, dict): - return "" - return str(message.get("content") or "") - - -def _extract_provider_stream_delta(data: Any) -> str: - if not isinstance(data, dict): - return "" - choices = data.get("choices") or [] - if not choices or not isinstance(choices[0], dict): - text = data.get("text") or data.get("content") - return str(text or "") - delta = choices[0].get("delta") or {} - if isinstance(delta, dict): - content = delta.get("content") - if content: - return str(content) - message = choices[0].get("message") or {} - if isinstance(message, dict): - content = message.get("content") - if content: - return str(content) - text = choices[0].get("text") or data.get("text") or data.get("content") - return str(text or "") - - -def _provider_response_meta(data: Any) -> Dict[str, Any]: - if not isinstance(data, dict): - return {} - choices = data.get("choices") or [] - first = choices[0] if choices and isinstance(choices[0], dict) else {} - return { - "usage": data.get("usage"), - "finish_reason": first.get("finish_reason"), - } - - - - -def _city_ai_response_example(unit: str) -> Dict[str, Any]: - return { - "metar_read_zh": f"最新观测显示 37.0{unit or '°C'}，观测时间 04:30Z；风和云量暂未显示强降温信号，后续观测用于确认升温路径。", - "metar_read_en": f"The latest observation shows 37.0{unit or '°C'} at 04:30Z; wind and cloud signals do not yet show a strong cooling break, so later observations should confirm the warming path.", - "taf_read_zh": "TAF 预报 14-16Z BECMG 18012KT，午后风向切换为海风可能抑制升温。", - "taf_read_en": "TAF shows BECMG 18012KT during 14-16Z; afternoon onshore wind shift may cap warming.", - "probability_read_zh": f"概率分布偏右，最高概率桶 42-43{unit or '°C'}（~35%），上方尾部延至 45{unit or '°C'}。", - "probability_read_en": f"Distribution skews right; peak bucket 42-43{unit or '°C'} (~35%), upper tail to 45{unit or '°C'}.", - "final_judgment_zh": f"预计最高温暂以 43.0{unit or '°C'} 附近为中枢。", - "final_judgment_en": f"The expected daily high is centered near 43.0{unit or '°C'}.", - "predicted_max": 43.0, - "range_low": 42.0, - "range_high": 44.0, - "unit": unit or "°C", - "confidence": "medium", - "reasoning_zh": "DEB 与多数模型集中在同一温区，最新观测仍处于上午升温路径。", - "reasoning_en": "DEB and most models cluster in the same temperature band, while the latest observation remains on a morning warming path.", - "risks_zh": ["若后续观测升温放缓或云雨增强，需要下调中枢。"], - "risks_en": ["If later observations show slower warming or stronger cloud/rain, revise the center lower."], - "model_cluster_note_zh": "7/8 个模型落在 DEB ±2°C 内，模型支撑较集中。", - "model_cluster_note_en": "7/8 models are within DEB ±2°C, so model support is clustered.", - } - - -def _city_ai_stream_response_example(unit: str) -> Dict[str, Any]: - return { - "metar_read_zh": f"最新 METAR 显示 37.0{unit or '°C'}，报文时间 04:30Z；风和云量暂未显示强降温信号，后续报文用于确认升温路径。", - "metar_read_en": f"The latest METAR shows 37.0{unit or '°C'} at 04:30Z; wind and cloud signals do not yet show a strong cooling break, so later reports should confirm the warming path.", - "taf_read_zh": "TAF 预报 14-16Z BECMG 18012KT，午后风向转为海风可能抑制升温，需关注。", - "taf_read_en": "TAF shows BECMG 18012KT during 14-16Z; afternoon onshore wind shift may cap further warming.", - "probability_read_zh": f"概率分布偏右，最高概率桶 42-43{unit or '°C'}（~35%），上方尾部延至 45{unit or '°C'}。", - "probability_read_en": f"Distribution skews right; peak bucket 42-43{unit or '°C'} (~35%), upper tail extends to 45{unit or '°C'}.", - "predicted_max": 43.0, - "range_low": 42.0, - "range_high": 44.0, - "unit": unit or "°C", - "confidence": "medium", - "final_judgment_zh": f"预计最高温暂以 43.0{unit or '°C'} 附近为中枢。", - "final_judgment_en": f"The expected daily high is centered near 43.0{unit or '°C'}.", - "reasoning_zh": "当前观测仍贴近上午升温路径，若后续风向转为海风或云雨增强，再下修最高温中枢。", - "reasoning_en": "The latest observation still fits the morning warming path; revise the daily-high center lower if later wind turns onshore or cloud/rain strengthens.", - } diff --git a/web/scan_city_ai_prompt.py b/web/scan_city_ai_prompt.py deleted file mode 100644 index 8ea17eaa..00000000 --- a/web/scan_city_ai_prompt.py +++ /dev/null @@ -1,261 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any, Dict - -from web.scan_city_ai_helpers import ( - CITY_AI_REQUIRED_FIELDS, - CITY_AI_STREAM_PROVIDER_FIELDS, - _city_ai_response_example, - _city_ai_stream_response_example, -) - -SCAN_CITY_AI_PROMPT_VERSION = "city-observation-read-v6" - - -def _normalize_locale(value: Any) -> str: - text = str(value or "").strip().lower() - return "en-US" if text.startswith("en") else "zh-CN" - - -def _observation_prompt_context(ai_input: Dict[str, Any]) -> Dict[str, Any]: - observation_anchor = ai_input.get("observation_anchor") if isinstance(ai_input.get("observation_anchor"), dict) else {} - is_airport_metar = observation_anchor.get("is_airport_metar") is not False - return { - "anchor": observation_anchor, - "is_airport_metar": is_airport_metar, - "read_label_zh": str(observation_anchor.get("read_label_zh") or ("机场报文解读" if is_airport_metar else "官方观测解读")), - "instruction_zh": str(observation_anchor.get("instruction_zh") or ""), - } - - -def build_city_ai_request_json( - ai_input: Dict[str, Any], - *, - locale: str, - model: str, - max_tokens: int, -) -> Dict[str, Any]: - normalized_locale = _normalize_locale(locale) - context = _observation_prompt_context(ai_input) - observation_label_zh = context["read_label_zh"] - observation_instruction = context["instruction_zh"] - system_prompt = ( - "你是 PolyWeather 的城市最高温 AI 预测员。你必须直接阅读用户给出的城市 JSON，" - "独立判断该城市今日最高温路径。不要写套利、交易、BUY YES/NO、价格、edge 或 Kelly。" - f"你的核心输出是：最终最高温点估计、置信区间、置信度、最终判断、{observation_label_zh}、判断依据和风险。" - "预测方法：首先查看 model_cluster.sources 中的全部模型（含 DEB）的集中区间和中位数作为基线；" - "然后重点阅读 metar_context 和 current 中的最新观测数据（温度趋势、风向风速、湿度、云量、能见度、露点），" - "根据实测信号独立判断基线应该是上修、下修还是维持，并在 predicted_max 中给出你的独立判断。" - "DEB 只是模型集群中的一个融合参考，不应该直接照搬为 predicted_max；" - "你必须综合所有模型 + 观测信号后给出自己的数字，与 DEB 有差异是正常的。" - f"{observation_instruction}" - "如果实测温度与模型集群走势出现偏差，要明确说明偏差方向和可能修正。" - "你可以基于城市、时间、季节、站点位置、风向/风速、云、能见度、露点等判断风或天气是否可能影响温度路径，" - "但必须使用「可能」「倾向」「需要确认」等非绝对表达。" - "观测解读必须具体：写清楚最新观测/报文时间、温度、风向风速、云量/天气、能见度或露点中与温度路径相关的因素。" - "涉及风时必须说明该风向对本城市/机场最高温路径倾向增温、降温还是中性，并给出理由；" - "不得只写「风向切换可能冷平流」，必须说明是哪一类风向或哪段风向切换可能带来冷/暖平流。" - "涉及 TAF 或云雨扰动时必须给出报文中的有效时间、BECMG/TEMPO/FM 时间窗或说明「未给出明确时间」；" - "如果没有 TAF 时间依据，不要笼统写「峰值窗口云雨扰动风险」。" - "如果峰值窗口尚未到来，不能过早下最终结论；如果峰值窗口已过或实测已创高，需要更重视最新实测。" - "所有面向用户的自然语言字段必须同时填写简体中文和英文两套内容：" - "_zh 字段写简体中文，_en 字段写英文。前端会按用户界面语言直接切换字段，不能留空。" - "risks 最多 2 条，每条必须包含触发条件或方向来源；reasoning、model_cluster_note 各 1 句，metar_read 用 1-2 句。" - "只返回 JSON object，不要 Markdown。" - ) - user_payload = { - "locale": normalized_locale, - "task": ( - "Return strict JSON with: predicted_max, range_low, range_high, unit, confidence, " - "final_judgment_zh, final_judgment_en, metar_read_zh, metar_read_en, taf_read_zh, taf_read_en, probability_read_zh, probability_read_en, " - "reasoning_zh, reasoning_en, risks_zh, risks_en, model_cluster_note_zh, model_cluster_note_en. " - "Fill every *_zh field in Simplified Chinese and every *_en field in English in the same response. " - "Use this exact JSON object shape; do not return an array, markdown, or prose outside JSON. " - "Keep final_judgment one short decision sentence. metar_read must explain the latest observation source " - "with report/observation time, temperature, wind direction/speed, cloud/weather/visibility/dewpoint if available. " - "taf_read must interpret TAF for today's peak window impact (BECMG/TEMPO timing, wind shifts). If no TAF, write '无可用 TAF'/'No TAF available'. " - "probability_read must describe the probability distribution shape in 1 sentence (peak bucket, skew). " - "For wind, explicitly say whether the current wind tends to warm, cool, or be neutral for today's high, " - "and why in local city/station context. If mentioning cold/warm advection, name the wind direction or " - "direction shift responsible. If mentioning TAF risk, include the concrete TAF time window or say no " - "explicit timing is available. model_cluster_note must state " - "how many model sources are available, the cluster range and median, whether the spread is tight or wide, " - "and whether you adjusted above or below the cluster median based on observation signals. " - "Keep the whole JSON compact." - ), - "required_json_keys": CITY_AI_REQUIRED_FIELDS, - "json_example": _city_ai_response_example(str(ai_input.get("temp_symbol") or "\u00b0C")), - "city_snapshot": ai_input, - } - return { - "model": model, - "temperature": 0.2, - "max_tokens": max_tokens, - "response_format": {"type": "json_object"}, - "messages": [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": json.dumps(user_payload, ensure_ascii=False), - }, - ], - "_polyweather_user_payload": user_payload, - "_polyweather_system_prompt": system_prompt, - } - - -def build_city_ai_empty_retry_request(request_json: Dict[str, Any]) -> Dict[str, Any]: - system_prompt = str(request_json.get("_polyweather_system_prompt") or "") - user_payload = request_json.get("_polyweather_user_payload") if isinstance(request_json.get("_polyweather_user_payload"), dict) else {} - retry_payload = { - key: value - for key, value in request_json.items() - if not key.startswith("_polyweather_") - } - retry_payload.pop("response_format", None) - retry_payload["temperature"] = 0.1 - retry_payload["messages"] = [ - { - "role": "system", - "content": ( - system_prompt - + " 这次重试必须返回一个紧凑 JSON object，不要解释，不要空回复。" - + " If you cannot infer a field, still return the field with a cautious sentence." - ), - }, - { - "role": "user", - "content": json.dumps( - { - **user_payload, - "retry_reason": "previous provider response had empty message.content", - "task": ( - str(user_payload.get("task") or "") - + " The previous response had empty content. Return only one compact JSON object now." - ), - }, - ensure_ascii=False, - ), - }, - ] - return retry_payload - - -def build_city_ai_repair_request_json( - *, - ai_input: Dict[str, Any], - locale: str, - model: str, - max_tokens: int, - previous_error: str, - previous_content: str, -) -> Dict[str, Any]: - normalized_locale = _normalize_locale(locale) - return { - "model": model, - "temperature": 0.0, - "max_tokens": min(max(max_tokens, 1200), 64000), - "response_format": {"type": "json_object"}, - "messages": [ - { - "role": "system", - "content": ( - "You repair PolyWeather AI output into one strict JSON object. " - "Do not add facts that are not present in the original city snapshot or previous assistant content. " - "Return only JSON, no markdown." - ), - }, - { - "role": "user", - "content": json.dumps( - { - "locale": normalized_locale, - "required_schema": CITY_AI_REQUIRED_FIELDS, - "previous_error": previous_error, - "previous_assistant_content": previous_content, - "city_snapshot": ai_input, - "instruction": ( - "Fill *_zh fields in Simplified Chinese and *_en fields in English; do not leave either language empty. " - "Make final_judgment one direct sentence about today's high temperature. " - "metar_read must interpret the latest observation source with report/observation time, temperature, " - "wind direction/speed, cloud/weather/visibility/dewpoint if available. State whether " - "the current wind tends to warm, cool, or stay neutral for the temperature path, and why. " - "If mentioning cold/warm advection or TAF risk, include the responsible wind direction " - "or the concrete TAF time window; otherwise say timing is not explicit. " - "model_cluster_note must mention available model count/range and whether it supports DEB. " - "Keep the JSON compact." - ), - }, - ensure_ascii=False, - ), - }, - ], - } - - -def build_city_ai_stream_request( - ai_input: Dict[str, Any], - *, - locale: str, - model: str, - max_tokens: int, -) -> Dict[str, Any]: - normalized_locale = _normalize_locale(locale) - context = _observation_prompt_context(ai_input) - is_airport_metar = context["is_airport_metar"] - role_label = "机场 METAR 解读与最高温预测员" if is_airport_metar else "官方观测站解读与最高温预测员" - source_instruction = context["instruction_zh"] - system_prompt = ( - f"你是 PolyWeather 的{role_label}。" - "只返回一个紧凑 JSON object，不要 Markdown。" - f"必须基于最新观测/报文独立判断该城市今日最高温，输出 metar_read_zh、metar_read_en、taf_read_zh、taf_read_en、probability_read_zh、probability_read_en、predicted_max、range_low、range_high、unit、confidence、final_judgment_zh、final_judgment_en、reasoning_zh、reasoning_en 字段；" - "模型一致性和风险清单由后端规则补齐，不要生成这些字段。" - f"预测方法：先看 model_cluster.sources 中各模型（含 DEB）的集中区间作为基线；" - "然后重点阅读 metar_context 和 current 中的观测信号——温度趋势、风向风速、湿度、云量、能见度——" - "独立判断 predicted_max 应该落在基线区间的哪个位置（偏上/偏下/中枢），不要直接照搬 DEB 的值。" - f"{source_instruction}" - "观测解读必须具体说明观测/报文时间、温度、风向风速、云量/天气/能见度/露点中与温度路径相关的因素；每个字段最多 1-2 句。" - "涉及风时要说明当前风向对站点最高温路径倾向增温、降温还是中性，并给出理由。" - "如果 observation_anchor.is_airport_metar 为 false，不得使用 METAR、TAF、机场报文等称谓。" - "predicted_max 是你的独立预测值（float），range_low/range_high 是预测区间，unit 为温度单位，confidence 为 low/medium/high。" - "final_judgment 用一句话给出今日最高温结论。reasoning 必须解释你相对于模型集群基线做了哪种修正及原因。" - "taf_read_zh/en: 如果 city_snapshot.taf 有有效内容，用 1-2 句解读机场预报中对今日峰值窗口有影响的变化（BECMG/TEMPO 时间窗、风向切换、云量变化）；如果无 TAF 或 TAF 不含今日白天有效时段，写「无可用 TAF」/「No TAF available」。" - "probability_read_zh/en: 如果 city_snapshot.probability 有分布数据，用 1 句描述概率分布形态——最高概率桶落在哪、分布偏左/偏右/对称。" - "所有 *_zh 字段写简体中文，所有 *_en 字段写英文，不得留空。" - "不要写交易建议、BUY/SELL、Kelly 或套利。" - ) - return { - "model": model, - "temperature": 0.2, - "max_tokens": max_tokens, - "response_format": {"type": "json_object"}, - "stream": True, - "messages": [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": json.dumps( - { - "locale": normalized_locale, - "task": ( - "Return JSON keys in this exact order: metar_read_zh, metar_read_en, taf_read_zh, taf_read_en, probability_read_zh, probability_read_en, predicted_max, range_low, range_high, unit, confidence, final_judgment_zh, final_judgment_en, reasoning_zh, reasoning_en. " - "predicted_max must be your independent float prediction, based on model cluster baseline adjusted by the latest METAR/observation signals. " - "Do not copy DEB directly \u2014 use the full model spread + your own reading of wind, cloud, temperature trend from the bulletin. " - "reasoning must explain what adjustment you made relative to the model cluster and why. " - "taf_read must interpret TAF for peak window impact if available. " - "probability_read must describe the probability distribution shape in 1 sentence. " - "Do not return risks or model_cluster_note. Keep it compact. " - "Return exactly one JSON object and no markdown." - ), - "required_json_keys": CITY_AI_STREAM_PROVIDER_FIELDS, - "json_example": _city_ai_stream_response_example( - str(ai_input.get("temp_symbol") or "\u00b0C") - ), - "city_snapshot": ai_input, - }, - ensure_ascii=False, - ), - }, - ], - } diff --git a/web/scan_city_ai_provider.py b/web/scan_city_ai_provider.py deleted file mode 100644 index 5bb34e0e..00000000 --- a/web/scan_city_ai_provider.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any, Dict - -import httpx -from loguru import logger - -from web.scan_city_ai_fallback import ( - _build_city_ai_fallback, - _complete_city_ai_payload, -) -from web.scan_city_ai_helpers import ( - _extract_ai_json_object, - _extract_provider_content, - _provider_response_meta, - _truncate_ai_text, -) -from web.scan_city_ai_prompt import ( - _normalize_locale, - build_city_ai_empty_retry_request, - build_city_ai_repair_request_json, - build_city_ai_request_json, -) - - -def _call_deepseek_city_ai( - ai_input: Dict[str, Any], - *, - locale: str = "zh-CN", - api_key: str, - base_url: str, - model: str, - max_tokens: int, - timeout_sec: int, -) -> Dict[str, Any]: - if not api_key: - raise RuntimeError("scan AI API key is not configured") - normalized_locale = _normalize_locale(locale) - timeout = httpx.Timeout( - timeout=float(timeout_sec), - connect=min(8.0, float(timeout_sec)), - read=float(timeout_sec), - write=10.0, - pool=5.0, - ) - request_json = build_city_ai_request_json( - ai_input, - locale=normalized_locale, - model=model, - max_tokens=max_tokens, - ) - provider_request = { - key: value - for key, value in request_json.items() - if not key.startswith("_polyweather_") - } - logger.info( - "scan city AI provider request city={} locale={} input_bytes={} max_tokens={} timeout_sec={}", - ai_input.get("city"), - normalized_locale, - len(json.dumps(provider_request, ensure_ascii=False, default=str).encode("utf-8")), - request_json.get("max_tokens"), - timeout_sec, - ) - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - } - with httpx.Client(timeout=timeout) as client: - response = client.post( - f"{base_url}/chat/completions", - headers=headers, - json=provider_request, - ) - response.raise_for_status() - data = response.json() - content = _extract_provider_content(data) - if not str(content or "").strip(): - logger.warning( - "scan city AI provider returned empty content city={} locale={} finish_reason={} retrying_without_json_mode=true", - ai_input.get("city"), - normalized_locale, - _provider_response_meta(data).get("finish_reason"), - ) - retry_payload = build_city_ai_empty_retry_request(request_json) - response = client.post( - f"{base_url}/chat/completions", - headers=headers, - json=retry_payload, - ) - response.raise_for_status() - data = response.json() - content = _extract_provider_content(data) - try: - parsed = _extract_ai_json_object(str(content or "")) - except ValueError as exc: - preview = _truncate_ai_text(content, 700) - logger.warning( - "scan city AI provider returned non-json city={} locale={} finish_reason={} content_preview={}", - ai_input.get("city"), - normalized_locale, - _provider_response_meta(data).get("finish_reason"), - preview, - ) - repair_payload = build_city_ai_repair_request_json( - ai_input=ai_input, - locale=normalized_locale, - model=model, - max_tokens=max_tokens, - previous_error=str(exc), - previous_content=_truncate_ai_text(content, 5000), - ) - try: - repair_response = client.post( - f"{base_url}/chat/completions", - headers=headers, - json=repair_payload, - ) - repair_response.raise_for_status() - repair_data = repair_response.json() - repair_content = _extract_provider_content(repair_data) - parsed = _extract_ai_json_object(str(repair_content or "")) - if isinstance(parsed, dict): - parsed["_polyweather_meta"] = { - **_provider_response_meta(repair_data), - "repaired_from_non_json": True, - "original_finish_reason": _provider_response_meta(data).get("finish_reason"), - "original_content_preview": preview, - } - return _complete_city_ai_payload( - parsed, - ai_input, - locale=normalized_locale, - ) - except Exception as repair_exc: - logger.warning( - "scan city AI provider json repair failed city={} locale={} error={}", - ai_input.get("city"), - normalized_locale, - repair_exc, - ) - return _build_city_ai_fallback( - ai_input, - locale=normalized_locale, - reason=str(exc), - raw_content=str(content or ""), - provider_data=data, - ) - if isinstance(data, dict): - parsed["_polyweather_meta"] = _provider_response_meta(data) - return _complete_city_ai_payload( - parsed, - ai_input, - locale=normalized_locale, - ) diff --git a/web/scan_terminal_ai_compact.py b/web/scan_terminal_ai_compact.py deleted file mode 100644 index 90c63d1a..00000000 --- a/web/scan_terminal_ai_compact.py +++ /dev/null @@ -1,466 +0,0 @@ -from __future__ import annotations - -import re -from datetime import datetime -from typing import Any, Dict, List, Optional - -from web.scan_city_ai_helpers import _safe_float, _truncate_ai_text - - -def _compact_ai_candidate(row: Dict[str, Any]) -> Dict[str, Any]: - return { - "id": row.get("id"), - "action": row.get("action"), - "side": row.get("side"), - "target_label": row.get("target_label"), - "target_value": row.get("target_value"), - "target_threshold": row.get("target_threshold"), - "target_unit": row.get("target_unit"), - "market_probability": row.get("market_probability"), - "market_event_probability": row.get("market_event_probability"), - "yes_ask": row.get("yes_ask"), - "no_ask": row.get("no_ask"), - "ask": row.get("ask"), - "spread": row.get("spread"), - "quote_age_ms": row.get("quote_age_ms"), - "cluster_role": row.get("cluster_role"), - "model_cluster_sources": _compact_ai_model_sources(row), - "metar_context": row.get("metar_context") or {}, - "window_phase": row.get("window_phase"), - "peak_window_label": row.get("peak_window_label"), - "minutes_until_peak_start": row.get("minutes_until_peak_start"), - "minutes_until_peak_end": row.get("minutes_until_peak_end"), - "trend_alignment": row.get("trend_alignment"), - "tradable": row.get("tradable"), - "accepting_orders": row.get("accepting_orders"), - } - - -def _normalize_ai_city_key(value: Any) -> str: - return str(value or "").strip().lower().replace(" ", "").replace("-", "").replace("_", "") - - -def _compact_ai_model_sources(row: Dict[str, Any]) -> List[Dict[str, Any]]: - raw_sources = row.get("model_cluster_sources") - if not isinstance(raw_sources, dict): - return [] - sources: List[Dict[str, Any]] = [] - for name, value in raw_sources.items(): - if _safe_float(value) is None: - continue - sources.append({"model": str(name), "value": value}) - return sources[:12] - - -def _observation_sort_key(point: Dict[str, Any]) -> tuple[int, str]: - raw_time = str(point.get("time") or "").strip() - try: - parsed = datetime.fromisoformat(raw_time.replace("Z", "+00:00")) - return parsed.hour * 60 + parsed.minute, raw_time - except Exception: - pass - match = re.search(r"(\d{1,2}):(\d{2})", raw_time) - if match: - hour = max(0, min(23, int(match.group(1)))) - minute = max(0, min(59, int(match.group(2)))) - return hour * 60 + minute, raw_time - return 9999, raw_time - - -def _compact_observation_points(raw_points: Any, limit: int = 24) -> List[Dict[str, Any]]: - if not isinstance(raw_points, list): - return [] - points: List[Dict[str, Any]] = [] - for item in raw_points: - if isinstance(item, dict): - temp = _safe_float(item.get("temp")) - time_value = str(item.get("time") or item.get("obs_time") or item.get("time_label") or "").strip() - elif isinstance(item, (list, tuple)) and len(item) >= 2: - time_value = str(item[0] or "").strip() - temp = _safe_float(item[1]) - else: - continue - if temp is None or not time_value: - continue - points.append({"time": time_value, "temp": temp}) - sorted_points = sorted(points, key=_observation_sort_key) - return sorted_points[-max(1, int(limit)) :] - - -def _compact_ai_text(value: Any, limit: int = 700) -> Optional[str]: - text = _truncate_ai_text(value, limit).strip() - return text or None - - -def _compact_hourly_context(raw_hourly: Any) -> Dict[str, Any]: - if not isinstance(raw_hourly, dict): - return {} - times = raw_hourly.get("times") or raw_hourly.get("time") or [] - temps = raw_hourly.get("temps") or raw_hourly.get("temperature_2m") or [] - radiation = raw_hourly.get("radiation") or raw_hourly.get("shortwave_radiation") or [] - if not isinstance(times, list) or not isinstance(temps, list): - return {} - - points: List[Dict[str, Any]] = [] - for idx, raw_time in enumerate(times): - temp = _safe_float(temps[idx] if idx < len(temps) else None) - if temp is None: - continue - time_text = str(raw_time or "").strip() - if "T" in time_text: - time_text = time_text.split("T", 1)[1][:5] - elif len(time_text) > 5: - time_text = time_text[:5] - point: Dict[str, Any] = {"time": time_text, "temp": temp} - rad = _safe_float(radiation[idx] if isinstance(radiation, list) and idx < len(radiation) else None) - if rad is not None: - point["radiation"] = rad - points.append(point) - - if not points: - return {} - max_point = max(points, key=lambda item: _safe_float(item.get("temp")) or -999.0) - sample_indexes = { - idx - for idx in range(len(points)) - if idx % 2 == 0 or idx >= len(points) - 4 or points[idx] is max_point - } - samples = [points[idx] for idx in sorted(sample_indexes)][-14:] - return { - "sample_count": len(points), - "forecast_hourly_max": max_point, - "samples": samples, - } - - -def _compact_taf_context(raw_taf_data: Any) -> Dict[str, Any]: - if not isinstance(raw_taf_data, dict): - return {} - signal = raw_taf_data.get("signal") if isinstance(raw_taf_data.get("signal"), dict) else {} - source = signal or raw_taf_data - raw_taf = raw_taf_data.get("raw_taf") or source.get("raw_taf") - compact: Dict[str, Any] = { - "available": bool(source.get("available") or raw_taf), - "raw_taf": _compact_ai_text(raw_taf, 900), - "issue_time": raw_taf_data.get("issue_time") or source.get("issue_time"), - "valid_time_from": raw_taf_data.get("valid_time_from") or source.get("valid_time_from"), - "valid_time_to": raw_taf_data.get("valid_time_to") or source.get("valid_time_to"), - "peak_window": source.get("peak_window"), - "suppression_level": source.get("suppression_level"), - "disruption_level": source.get("disruption_level"), - "wind_shift": source.get("wind_shift"), - "wind_regimes": source.get("wind_regimes"), - "summary_zh": _compact_ai_text(source.get("summary_zh"), 260), - "summary_en": _compact_ai_text(source.get("summary_en"), 260), - } - segments = source.get("segments") if isinstance(source.get("segments"), list) else [] - markers = source.get("markers") if isinstance(source.get("markers"), list) else [] - if segments: - compact["segments"] = segments[:3] - if markers: - compact["markers"] = markers[:4] - return {key: value for key, value in compact.items() if value not in (None, "", [])} - - -def _compact_vertical_context(raw_vertical: Any) -> Dict[str, Any]: - if not isinstance(raw_vertical, dict): - return {} - keys = [ - "source", - "window_start", - "window_end", - "suppression_risk", - "trigger_risk", - "mixing_strength", - "shear_risk", - "heating_setup", - "heating_score", - "summary_zh", - "summary_en", - ] - compact: Dict[str, Any] = {} - for key in keys: - value = raw_vertical.get(key) - if isinstance(value, str): - value = _compact_ai_text(value, 280) - if value not in (None, "", []): - compact[key] = value - return compact - - -def _compact_intraday_context(raw_intraday: Any) -> Dict[str, Any]: - if not isinstance(raw_intraday, dict): - return {} - compact: Dict[str, Any] = {} - for key in [ - "headline", - "headline_en", - "confidence", - "base_case_bucket", - "upside_bucket", - "downside_bucket", - "next_observation_time", - "peak_window", - ]: - value = raw_intraday.get(key) - if isinstance(value, str): - value = _compact_ai_text(value, 220) - if value not in (None, "", []): - compact[key] = value - signals = raw_intraday.get("signal_contributions") - if isinstance(signals, list): - compact["signal_contributions"] = [ - { - "label": item.get("label"), - "label_en": item.get("label_en"), - "direction": item.get("direction"), - "strength": item.get("strength"), - "summary": _compact_ai_text(item.get("summary"), 180), - "summary_en": _compact_ai_text(item.get("summary_en"), 180), - } - for item in signals[:4] - if isinstance(item, dict) - ] - return compact - - -def _build_metar_decision_context(data: Dict[str, Any]) -> Dict[str, Any]: - today_obs = _compact_observation_points(data.get("metar_today_obs"), 36) - recent_obs = _compact_observation_points(data.get("metar_recent_obs"), 12) - settlement_obs = _compact_observation_points(data.get("settlement_today_obs"), 36) - airport_current = data.get("airport_current") if isinstance(data.get("airport_current"), dict) else {} - metar_status = data.get("metar_status") if isinstance(data.get("metar_status"), dict) else {} - - source_obs = today_obs or recent_obs or settlement_obs - trend_source = recent_obs or source_obs[-4:] - last_point = source_obs[-1] if source_obs else {} - first_trend = trend_source[0] if trend_source else {} - last_trend = trend_source[-1] if trend_source else {} - max_point = None - for point in source_obs: - if max_point is None or float(point["temp"]) >= float(max_point["temp"]): - max_point = point - - last_temp = _safe_float(last_point.get("temp")) - first_temp = _safe_float(first_trend.get("temp")) - trend_last_temp = _safe_float(last_trend.get("temp")) - trend_delta = ( - trend_last_temp - first_temp - if trend_last_temp is not None and first_temp is not None and len(trend_source) >= 2 - else None - ) - station = data.get("risk") if isinstance(data.get("risk"), dict) else {} - current = data.get("current") if isinstance(data.get("current"), dict) else {} - settlement_station = data.get("settlement_station") if isinstance(data.get("settlement_station"), dict) else {} - settlement_source = str( - current.get("settlement_source") - or settlement_station.get("settlement_source") - or "metar" - ).strip().lower() - is_hko = settlement_source == "hko" - source_label = "HKO" if is_hko else "METAR" - return { - "source": source_label, - "is_airport_metar": not is_hko, - "station": ( - current.get("station_code") - or settlement_station.get("settlement_station_code") - or station.get("icao") - or airport_current.get("station_code") - ), - "station_label": ( - current.get("station_name") - or settlement_station.get("settlement_station_label") - or station.get("airport") - or airport_current.get("station_label") - ), - "today_obs": today_obs[-12:], - "recent_obs": recent_obs[-8:], - "settlement_today_obs": settlement_obs[-12:], - "obs_count": len(source_obs), - "last_time": last_point.get("time"), - "last_temp": last_temp, - "max_temp": _safe_float((max_point or {}).get("temp")), - "max_time": (max_point or {}).get("time"), - "trend_delta": trend_delta, - "stale_for_today": bool(metar_status.get("stale_for_today")), - "available_for_today": bool(metar_status.get("available_for_today")), - "last_observation_time": metar_status.get("last_observation_time"), - "airport_current_temp": _safe_float(airport_current.get("temp")), - "airport_max_so_far": _safe_float(airport_current.get("max_so_far")), - "airport_obs_time": airport_current.get("obs_time"), - "airport_report_time": airport_current.get("report_time"), - "airport_raw_metar": airport_current.get("raw_metar"), - "airport_wx_desc": airport_current.get("wx_desc"), - "airport_cloud_desc": airport_current.get("cloud_desc"), - "airport_visibility_mi": _safe_float(airport_current.get("visibility_mi")), - "airport_wind_speed_kt": _safe_float(airport_current.get("wind_speed_kt")), - "airport_wind_dir": _safe_float(airport_current.get("wind_dir")), - "airport_humidity": _safe_float(airport_current.get("humidity")), - } - - -def _city_observation_anchor(data: Dict[str, Any]) -> Dict[str, Any]: - current = data.get("current") if isinstance(data.get("current"), dict) else {} - settlement_station = data.get("settlement_station") if isinstance(data.get("settlement_station"), dict) else {} - airport_current = data.get("airport_current") if isinstance(data.get("airport_current"), dict) else {} - risk = data.get("risk") if isinstance(data.get("risk"), dict) else {} - source = str( - current.get("settlement_source") - or settlement_station.get("settlement_source") - or "metar" - ).strip().lower() - is_hko = source == "hko" - if is_hko: - station_code = ( - current.get("station_code") - or settlement_station.get("settlement_station_code") - or "HKO" - ) - station_label = ( - current.get("station_name") - or settlement_station.get("settlement_station_label") - or "Hong Kong Observatory" - ) - return { - "source": "hko", - "source_label": "Hong Kong Observatory", - "is_airport_metar": False, - "station_code": station_code, - "station_label": station_label, - "read_label_zh": "香港天文台观测解读", - "read_label_en": "Hong Kong Observatory observation read", - "instruction_zh": ( - "该城市使用香港天文台/HKO 官方站点观测，提供温度、今日最高/最低温、相对湿度、" - "10分钟平均风速和风向数据。它不是机场 METAR，不得使用 METAR、TAF、机场报文、报文时间等称谓，" - "应使用「观测时间」「风速」「风向」「湿度」等官方气象站用语。" - "观测解读必须具体说明：最新观测时间、温度、风向风速对升温路径的影响（增温/降温/中性）、" - "湿度是否可能压制日间升温。涉及风时必须说明具体风向及来源（如「偏北风」「海风/陆风」）。" - "如果有今日最高温和最低温数据，应结合判断温度走势。" - ), - "instruction_en": ( - "This city uses Hong Kong Observatory/HKO official station observations, providing temperature, " - "today's high/low, relative humidity, 10-min mean wind speed and direction. " - "It is NOT an airport METAR; do not use METAR, TAF, airport bulletin, or report time terminology. " - "Use terms like observation time, wind speed, wind direction, humidity. " - "The observation read must be specific: latest observation time, temperature, whether the wind " - "direction tends to warm, cool or be neutral for today's high path and why, and whether humidity " - "may suppress daytime heating. When mentioning wind, specify the direction and source " - "(e.g. northerly, sea breeze/land breeze). If today's high/low are available, use them to assess the trend." - ), - } - return { - "source": "metar", - "source_label": "METAR", - "is_airport_metar": True, - "station_code": risk.get("icao") or airport_current.get("station_code"), - "station_label": risk.get("airport") or airport_current.get("station_label"), - "read_label_zh": "机场报文解读", - "read_label_en": "airport-bulletin read", - "instruction_zh": "该城市使用机场 METAR/TAF 作为日内实况证据。", - "instruction_en": "This city uses airport METAR/TAF as intraday observation evidence.", - } - - -def _compact_ai_city_group(rows: List[Dict[str, Any]]) -> Dict[str, Any]: - first = rows[0] - return { - "city": first.get("city"), - "city_display_name": first.get("city_display_name") or first.get("display_name") or first.get("city"), - "selected_date": first.get("selected_date") or first.get("local_date"), - "local_time": first.get("local_time"), - "temp_symbol": first.get("temp_symbol") or first.get("target_unit"), - "current_temp": first.get("current_temp"), - "current_max_so_far": first.get("current_max_so_far"), - "window_phase": first.get("window_phase"), - "remaining_window_minutes": first.get("remaining_window_minutes"), - "peak_window_label": first.get("peak_window_label"), - "minutes_until_peak_start": first.get("minutes_until_peak_start"), - "minutes_until_peak_end": first.get("minutes_until_peak_end"), - "metar_context": first.get("metar_context") or {}, - "model_cluster": { - "core_low": first.get("cluster_core_low"), - "core_high": first.get("cluster_core_high"), - "median": first.get("cluster_median"), - "deb_reference": first.get("cluster_deb_reference"), - "model_count": first.get("cluster_model_count"), - "sources": _compact_ai_model_sources(first), - }, - "contracts": [_compact_ai_candidate(row) for row in rows], - } - - -def build_scan_ai_prompt(payload: Dict[str, Any], *, max_rows: int) -> Dict[str, Any]: - raw_rows = [ - row - for row in (payload.get("rows") or [])[:max_rows] - if isinstance(row, dict) and row.get("id") - ] - grouped: Dict[str, List[Dict[str, Any]]] = {} - for row in raw_rows: - key = "|".join( - [ - _normalize_ai_city_key(row.get("city") or row.get("city_display_name")), - str(row.get("selected_date") or row.get("local_date") or ""), - ] - ) - grouped.setdefault(key, []).append(row) - cities = [_compact_ai_city_group(rows) for rows in grouped.values() if rows] - sent_contracts = sum(len(city.get("contracts") or []) for city in cities) - return { - "schema_version": "city_forecast_v1", - "snapshot_id": payload.get("snapshot_id"), - "generated_at": payload.get("generated_at"), - "summary": payload.get("summary") or {}, - "filters": payload.get("filters") or {}, - "city_count": len(cities), - "candidate_row_count": len(raw_rows), - "cities": cities, - "_polyweather_input_meta": { - "sent_cities": len(cities), - "sent_contracts": sent_contracts, - }, - } - - -def _compact_probability_context(probabilities: Any, deb: Any, unit: str) -> dict: - if not isinstance(probabilities, dict): - return {} - dist = probabilities.get("distribution") - if not isinstance(dist, list) or not dist: - return {} - top_buckets = sorted( - [b for b in dist if isinstance(b, dict) and b.get("probability")], - key=lambda b: float(b.get("probability", 0)), - reverse=True, - )[:3] - compact = { - "top_buckets": [ - { - "label": b.get("label", ""), - "prob": round(float(b.get("probability", 0)) * 100), - } - for b in top_buckets - ], - } - mu = probabilities.get("mu") - if mu is not None: - compact["mu"] = mu - spread = ( - round(float(probabilities.get("calibrated_sigma") or 0), 1) - or round(float(probabilities.get("raw_sigma") or 0), 1) - or None - ) - if spread is not None: - compact["sigma"] = spread - deb_val = deb.get("prediction") if isinstance(deb, dict) else None - if deb_val is not None and mu is not None: - if deb_val > mu: - compact["skew"] = "right" - elif deb_val < mu: - compact["skew"] = "left" - else: - compact["skew"] = "centered" - if unit: - compact["unit"] = unit - return compact diff --git a/web/scan_terminal_ai_merge.py b/web/scan_terminal_ai_merge.py deleted file mode 100644 index 50df832a..00000000 --- a/web/scan_terminal_ai_merge.py +++ /dev/null @@ -1,256 +0,0 @@ -from __future__ import annotations - -from datetime import datetime -from typing import Any, Dict, List, Optional - -from web.scan_city_ai_helpers import _safe_float -from web.scan_terminal_ai_compact import _normalize_ai_city_key -from web.scan_terminal_filters import safe_int as _safe_int -from web.scan_terminal_metar_gate import _apply_metar_gate_to_row - - -def _normalize_ai_items(raw_items: Any) -> List[Dict[str, Any]]: - if not isinstance(raw_items, list): - return [] - out: List[Dict[str, Any]] = [] - for item in raw_items: - if isinstance(item, str): - out.append({"row_id": item}) - elif isinstance(item, dict): - row_id = str(item.get("row_id") or item.get("id") or "").strip() - if row_id: - out.append({**item, "row_id": row_id}) - return out - - -def _normalize_ai_city_theses(raw_items: Any) -> List[Dict[str, Any]]: - if not isinstance(raw_items, list): - return [] - out: List[Dict[str, Any]] = [] - for item in raw_items: - if not isinstance(item, dict): - continue - city = str(item.get("city") or item.get("city_name") or "").strip() - if not city: - continue - out.append({**item, "city": city}) - return out - - -def _normalize_ai_city_forecasts(ai_raw: Dict[str, Any]) -> List[Dict[str, Any]]: - raw_items = ( - ai_raw.get("city_forecasts") - or ai_raw.get("city_predictions") - or ai_raw.get("city_max_forecasts") - or ai_raw.get("city_theses") - ) - if not isinstance(raw_items, list): - return [] - out: List[Dict[str, Any]] = [] - for item in raw_items: - if not isinstance(item, dict): - continue - city = str(item.get("city") or item.get("city_name") or "").strip() - if not city: - continue - predicted = ( - item.get("predicted_max") - if item.get("predicted_max") is not None - else item.get("max_temp") - if item.get("max_temp") is not None - else item.get("prediction") - ) - out.append( - { - **item, - "city": city, - "predicted_max": predicted, - "range_low": item.get("range_low") if item.get("range_low") is not None else item.get("low"), - "range_high": item.get("range_high") if item.get("range_high") is not None else item.get("high"), - "reasoning_zh": item.get("reasoning_zh") or item.get("thesis_zh") or item.get("summary_zh"), - "reasoning_en": item.get("reasoning_en") or item.get("thesis_en") or item.get("summary_en"), - } - ) - return out - - -def merge_scan_ai_result( - payload: Dict[str, Any], - ai_raw: Dict[str, Any], - *, - model: str, - max_rows: int, - timeout_sec: int, - cache_ttl_sec: int, - base_url: str, - cached: bool = False, - provider: str = "openai-compatible", - duration_ms: Optional[int] = None, - input_rows: Optional[int] = None, -) -> Dict[str, Any]: - rows = [dict(row) for row in (payload.get("rows") or []) if isinstance(row, dict)] - by_id = {str(row.get("id")): row for row in rows if row.get("id")} - recommendations = _normalize_ai_items(ai_raw.get("recommendations")) - vetoed = _normalize_ai_items(ai_raw.get("vetoed")) - downgraded = _normalize_ai_items(ai_raw.get("downgraded")) - watchlist = _normalize_ai_items(ai_raw.get("watchlist")) - city_theses = _normalize_ai_city_theses(ai_raw.get("city_theses")) - city_forecasts = _normalize_ai_city_forecasts(ai_raw) - contract_notes = _normalize_ai_items(ai_raw.get("contract_notes")) - - veto_ids = {str(item.get("row_id")) for item in vetoed} - downgrade_ids = {str(item.get("row_id")) for item in downgraded} - recommended_ids: set[str] = set() - watchlist_ids = {str(item.get("row_id")) for item in watchlist} - - thesis_by_city: Dict[str, Dict[str, Any]] = {} - for item in city_theses: - key = _normalize_ai_city_key(item.get("city")) - if key: - thesis_by_city[key] = item - forecast_by_city: Dict[str, Dict[str, Any]] = {} - for item in city_forecasts: - key = _normalize_ai_city_key(item.get("city")) - if key: - forecast_by_city[key] = item - - for row in rows: - city_key = _normalize_ai_city_key(row.get("city")) - display_key = _normalize_ai_city_key(row.get("city_display_name")) - thesis = thesis_by_city.get(city_key) or thesis_by_city.get(display_key) - forecast = forecast_by_city.get(city_key) or forecast_by_city.get(display_key) - if thesis: - row["ai_city_thesis_zh"] = thesis.get("thesis_zh") or thesis.get("summary_zh") - row["ai_city_thesis_en"] = thesis.get("thesis_en") or thesis.get("summary_en") - row["ai_city_confidence"] = thesis.get("confidence") - row["ai_city_model_cluster_note"] = thesis.get("model_cluster_note") - if forecast: - row["ai_predicted_max"] = _safe_float(forecast.get("predicted_max")) - row["ai_predicted_low"] = _safe_float(forecast.get("range_low")) - row["ai_predicted_high"] = _safe_float(forecast.get("range_high")) - row["ai_forecast_unit"] = forecast.get("unit") or row.get("temp_symbol") - row["ai_forecast_confidence"] = forecast.get("confidence") - row["ai_peak_window_zh"] = forecast.get("peak_window_zh") - row["ai_peak_window_en"] = forecast.get("peak_window_en") - row["ai_airport_metar_read_zh"] = forecast.get("metar_read_zh") - row["ai_airport_metar_read_en"] = forecast.get("metar_read_en") - row["ai_forecast_reason_zh"] = forecast.get("reasoning_zh") - row["ai_forecast_reason_en"] = forecast.get("reasoning_en") - row["ai_city_model_cluster_note"] = forecast.get("model_cluster_note") or row.get("ai_city_model_cluster_note") - row["ai_city_thesis_zh"] = row.get("ai_city_thesis_zh") or forecast.get("reasoning_zh") - row["ai_city_thesis_en"] = row.get("ai_city_thesis_en") or forecast.get("reasoning_en") - - for item in contract_notes: - row = by_id.get(str(item.get("row_id"))) - if not row: - continue - row["ai_forecast_match"] = item.get("forecast_match") or item.get("match") - row["ai_forecast_match_reason_zh"] = item.get("reason_zh") or item.get("reason") - row["ai_forecast_match_reason_en"] = item.get("reason_en") - - for item in vetoed: - row = by_id.get(str(item.get("row_id"))) - if not row: - continue - row["ai_decision"] = "veto" - row["ai_reason_zh"] = item.get("reason_zh") or item.get("reason") - row["ai_reason_en"] = item.get("reason_en") - for item in downgraded: - row = by_id.get(str(item.get("row_id"))) - if not row: - continue - row["ai_decision"] = "downgrade" - row["ai_reason_zh"] = item.get("reason_zh") or item.get("reason") - row["ai_reason_en"] = item.get("reason_en") - for item in watchlist: - row = by_id.get(str(item.get("row_id"))) - if not row: - continue - row["ai_watchlist_reason_zh"] = item.get("reason_zh") or item.get("reason") - row["ai_watchlist_reason_en"] = item.get("reason_en") - for fallback_rank, item in enumerate(recommendations, start=1): - row_id = str(item.get("row_id")) - row = by_id.get(row_id) - if not row: - continue - if row_id in veto_ids: - continue - recommended_ids.add(row_id) - row["ai_decision"] = str(item.get("decision") or "approve").strip().lower() or "approve" - row["ai_rank"] = _safe_int(item.get("rank"), fallback_rank) - row["ai_confidence"] = item.get("confidence") - row["ai_reason_zh"] = item.get("reason_zh") or item.get("reason") - row["ai_reason_en"] = item.get("reason_en") - row["ai_model_cluster_note"] = item.get("model_cluster_note") - - for row in rows: - row_id = str(row.get("id")) - if row_id not in recommended_ids and row_id not in veto_ids and row_id not in downgrade_ids: - row["ai_decision"] = row.get("ai_decision") or "neutral" - if row_id in watchlist_ids and row.get("ai_decision") == "neutral": - row["ai_decision"] = "watchlist" - _apply_metar_gate_to_row(row) - - def _ai_sort_key(row: Dict[str, Any]) -> tuple: - decision = str(row.get("ai_decision") or "").lower() - if decision == "veto": - tier = 3 - elif decision == "downgrade": - tier = 2 - elif row.get("ai_rank") is not None: - tier = 0 - else: - tier = 1 - return ( - tier, - _safe_int(row.get("ai_rank"), 999), - -float(row.get("final_score") or 0.0), - -float(row.get("edge_percent") or 0.0), - ) - - rows.sort(key=_ai_sort_key) - top_signal = next( - (row for row in rows if str(row.get("ai_decision") or "").lower() != "veto"), - rows[0] if rows else None, - ) - input_meta = ai_raw.get("_polyweather_input_meta") - sent_cities = input_meta.get("sent_cities") if isinstance(input_meta, dict) else None - sent_contracts = input_meta.get("sent_contracts") if isinstance(input_meta, dict) else None - ai_scan = { - "status": "ready", - "stage": "completed", - "model": model, - "cached": cached, - "generated_at": datetime.utcnow().isoformat() + "Z", - "snapshot_id": payload.get("snapshot_id"), - "input_rows": input_rows if input_rows is not None else len(payload.get("rows") or []), - "sent_rows": sent_contracts if sent_contracts is not None else min(len(payload.get("rows") or []), max_rows), - "sent_cities": sent_cities, - "sent_contracts": sent_contracts, - "duration_ms": duration_ms, - "timeout_sec": timeout_sec, - "cache_ttl_sec": cache_ttl_sec, - "provider": provider, - "base_url": base_url, - "summary_zh": ai_raw.get("summary_zh"), - "summary_en": ai_raw.get("summary_en"), - "city_forecasts": city_forecasts, - "contract_notes": contract_notes, - "city_theses": city_theses, - "watchlist": watchlist, - "recommended_count": sum(1 for row in rows if row.get("ai_rank") is not None), - "vetoed_count": sum(1 for row in rows if row.get("ai_decision") == "veto"), - "downgraded_count": sum(1 for row in rows if row.get("ai_decision") == "downgrade"), - "watchlist_count": sum(1 for row in rows if row.get("ai_decision") == "watchlist"), - } - meta = ai_raw.get("_polyweather_meta") - if isinstance(meta, dict): - ai_scan["usage"] = meta.get("usage") - ai_scan["finish_reason"] = meta.get("finish_reason") - merged = { - **payload, - "rows": rows, - "top_signal": top_signal, - "ai_scan": ai_scan, - } - return merged diff --git a/web/scan_terminal_cache.py b/web/scan_terminal_cache.py index 0e031cea..b7c92151 100644 --- a/web/scan_terminal_cache.py +++ b/web/scan_terminal_cache.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib import json import threading import time @@ -10,8 +9,6 @@ from typing import Any, Dict, Optional _SCAN_TERMINAL_CACHE_LOCK = threading.Lock() _SCAN_TERMINAL_CACHE: Dict[str, Dict[str, Any]] = {} _SCAN_TERMINAL_REFRESHING: set[str] = set() -_SCAN_TERMINAL_AI_CACHE_LOCK = threading.Lock() -_SCAN_TERMINAL_AI_CACHE: Dict[str, Dict[str, Any]] = {} def scan_terminal_cache_key(filters: Dict[str, Any]) -> str: @@ -91,73 +88,3 @@ def clear_scan_terminal_refreshing(filters: Dict[str, Any]) -> None: cache_key = scan_terminal_cache_key(filters) with _SCAN_TERMINAL_CACHE_LOCK: _SCAN_TERMINAL_REFRESHING.discard(cache_key) - - -def scan_ai_cache_key( - snapshot_id: str, - filters: Dict[str, Any], - *, - max_rows: int, - model: str, -) -> str: - raw = json.dumps( - { - "schema_version": "city_forecast_v1", - "snapshot_id": snapshot_id, - "filters": filters, - "model": model, - "max_rows": max_rows, - }, - sort_keys=True, - ensure_ascii=False, - ) - return hashlib.sha256(raw.encode("utf-8")).hexdigest() - - -def get_cached_scan_ai_result( - snapshot_id: str, - filters: Dict[str, Any], - *, - max_rows: int, - model: str, - ttl_sec: int, -) -> Optional[Dict[str, Any]]: - cache_key = scan_ai_cache_key( - snapshot_id, - filters, - max_rows=max_rows, - model=model, - ) - now = time.time() - with _SCAN_TERMINAL_AI_CACHE_LOCK: - cached = _SCAN_TERMINAL_AI_CACHE.get(cache_key) - if not cached: - return None - cached_at = float(cached.get("cached_at") or 0.0) - if now - cached_at >= float(ttl_sec): - return None - result = cached.get("result") - if isinstance(result, dict): - return dict(result) - return None - - -def set_cached_scan_ai_result( - snapshot_id: str, - filters: Dict[str, Any], - result: Dict[str, Any], - *, - max_rows: int, - model: str, -) -> None: - cache_key = scan_ai_cache_key( - snapshot_id, - filters, - max_rows=max_rows, - model=model, - ) - with _SCAN_TERMINAL_AI_CACHE_LOCK: - _SCAN_TERMINAL_AI_CACHE[cache_key] = { - "cached_at": time.time(), - "result": result, - } diff --git a/web/scan_terminal_city_row.py b/web/scan_terminal_city_row.py index ff5e03b3..98523629 100644 --- a/web/scan_terminal_city_row.py +++ b/web/scan_terminal_city_row.py @@ -1,13 +1,12 @@ from __future__ import annotations import hashlib +import re from datetime import datetime, timedelta from typing import Any, Dict, List, Optional -from web.core import CITIES +from web.core import CITIES, _sf as _safe_float from web.analysis_service import _analyze -from web.scan_city_ai_helpers import _safe_float -from web.scan_terminal_ai_compact import _build_metar_decision_context from web.scan_terminal_filters import ( market_region_from_tz_offset as _market_region_from_tz_offset, safe_int as _safe_int, @@ -226,3 +225,117 @@ def _build_quick_row( row["model_probability"] = best_model_prob row["final_score"] = float(deb.get("prediction") or 0) return row + + +# ── METAR/observation context helpers (moved from deleted scan_terminal_ai_compact) ── + + +def _observation_sort_key(point: Dict[str, Any]) -> tuple[int, str]: + raw_time = str(point.get("time") or "").strip() + try: + parsed = datetime.fromisoformat(raw_time.replace("Z", "+00:00")) + return parsed.hour * 60 + parsed.minute, raw_time + except Exception: + pass + match = re.search(r"(\d{1,2}):(\d{2})", raw_time) + if match: + hour = max(0, min(23, int(match.group(1)))) + minute = max(0, min(59, int(match.group(2)))) + return hour * 60 + minute, raw_time + return 9999, raw_time + + +def _compact_observation_points(raw_points: Any, limit: int = 24) -> List[Dict[str, Any]]: + if not isinstance(raw_points, list): + return [] + points: List[Dict[str, Any]] = [] + for item in raw_points: + if isinstance(item, dict): + temp = _safe_float(item.get("temp")) + time_value = str(item.get("time") or item.get("obs_time") or item.get("time_label") or "").strip() + elif isinstance(item, (list, tuple)) and len(item) >= 2: + time_value = str(item[0] or "").strip() + temp = _safe_float(item[1]) + else: + continue + if temp is None or not time_value: + continue + points.append({"time": time_value, "temp": temp}) + sorted_points = sorted(points, key=_observation_sort_key) + return sorted_points[-max(1, int(limit)):] + + +def _build_metar_decision_context(data: Dict[str, Any]) -> Dict[str, Any]: + today_obs = _compact_observation_points(data.get("metar_today_obs"), 36) + recent_obs = _compact_observation_points(data.get("metar_recent_obs"), 12) + settlement_obs = _compact_observation_points(data.get("settlement_today_obs"), 36) + airport_current = data.get("airport_current") if isinstance(data.get("airport_current"), dict) else {} + metar_status = data.get("metar_status") if isinstance(data.get("metar_status"), dict) else {} + + source_obs = today_obs or recent_obs or settlement_obs + trend_source = recent_obs or source_obs[-4:] + last_point = source_obs[-1] if source_obs else {} + first_trend = trend_source[0] if trend_source else {} + last_trend = trend_source[-1] if trend_source else {} + max_point = None + for point in source_obs: + if max_point is None or float(point["temp"]) >= float(max_point["temp"]): + max_point = point + + last_temp = _safe_float(last_point.get("temp")) + first_temp = _safe_float(first_trend.get("temp")) + trend_last_temp = _safe_float(last_trend.get("temp")) + trend_delta = ( + trend_last_temp - first_temp + if trend_last_temp is not None and first_temp is not None and len(trend_source) >= 2 + else None + ) + station = data.get("risk") if isinstance(data.get("risk"), dict) else {} + current = data.get("current") if isinstance(data.get("current"), dict) else {} + settlement_station = data.get("settlement_station") if isinstance(data.get("settlement_station"), dict) else {} + settlement_source = str( + current.get("settlement_source") + or settlement_station.get("settlement_source") + or "metar" + ).strip().lower() + is_hko = settlement_source == "hko" + source_label = "HKO" if is_hko else "METAR" + return { + "source": source_label, + "is_airport_metar": not is_hko, + "station": ( + current.get("station_code") + or settlement_station.get("settlement_station_code") + or station.get("icao") + or airport_current.get("station_code") + ), + "station_label": ( + current.get("station_name") + or settlement_station.get("settlement_station_label") + or station.get("airport") + or airport_current.get("station_label") + ), + "today_obs": today_obs[-12:], + "recent_obs": recent_obs[-8:], + "settlement_today_obs": settlement_obs[-12:], + "obs_count": len(source_obs), + "last_time": last_point.get("time"), + "last_temp": last_temp, + "max_temp": _safe_float((max_point or {}).get("temp")), + "max_time": (max_point or {}).get("time"), + "trend_delta": trend_delta, + "stale_for_today": bool(metar_status.get("stale_for_today")), + "available_for_today": bool(metar_status.get("available_for_today")), + "last_observation_time": metar_status.get("last_observation_time"), + "airport_current_temp": _safe_float(airport_current.get("temp")), + "airport_max_so_far": _safe_float(airport_current.get("max_so_far")), + "airport_obs_time": airport_current.get("obs_time"), + "airport_report_time": airport_current.get("report_time"), + "airport_raw_metar": airport_current.get("raw_metar"), + "airport_wx_desc": airport_current.get("wx_desc"), + "airport_cloud_desc": airport_current.get("cloud_desc"), + "airport_visibility_mi": _safe_float(airport_current.get("visibility_mi")), + "airport_wind_speed_kt": _safe_float(airport_current.get("wind_speed_kt")), + "airport_wind_dir": _safe_float(airport_current.get("wind_dir")), + "airport_humidity": _safe_float(airport_current.get("humidity")), + } diff --git a/web/scan_terminal_filters.py b/web/scan_terminal_filters.py index 0e8bd77d..b08ab257 100644 --- a/web/scan_terminal_filters.py +++ b/web/scan_terminal_filters.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import Any, Dict, Optional -from web.scan_city_ai_helpers import _safe_float +from web.core import _sf as _safe_float def safe_int(value: Any, default: int) -> int: diff --git a/web/scan_terminal_metar_gate.py b/web/scan_terminal_metar_gate.py index 715dea7e..e37f1b50 100644 --- a/web/scan_terminal_metar_gate.py +++ b/web/scan_terminal_metar_gate.py @@ -3,7 +3,7 @@ from __future__ import annotations import re from typing import Any, Dict, Optional -from web.scan_city_ai_helpers import _safe_float +from web.core import _sf as _safe_float from web.scan_terminal_filters import safe_int as _safe_int diff --git a/web/scan_terminal_ranker.py b/web/scan_terminal_ranker.py index 7d4bbf11..a6417e1b 100644 --- a/web/scan_terminal_ranker.py +++ b/web/scan_terminal_ranker.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import Any, Dict, List, Optional -from web.scan_city_ai_helpers import _safe_float +from web.core import _sf as _safe_float def build_ranked_scan_terminal_result( diff --git a/web/scan_terminal_service.py b/web/scan_terminal_service.py index e8d5e673..56716104 100644 --- a/web/scan_terminal_service.py +++ b/web/scan_terminal_service.py @@ -1,81 +1,32 @@ from __future__ import annotations -import json import re import threading import time -import hashlib from concurrent.futures import TimeoutError as FutureTimeoutError from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime -from typing import Any, Dict, Iterator, List, Optional +from typing import Any, Dict, List, Optional -import httpx from loguru import logger from web.analysis_service import _analyze from web.core import CITIES from web.services.scan_ai_config import ( - _SCAN_CITY_AI_CACHE, - _SCAN_CITY_AI_CACHE_LOCK, - _scan_ai_api_key, - SCAN_AI_API_KEY_ENV_HINT, - SCAN_AI_BASE_URL, - SCAN_AI_CACHE_TTL_SEC, - SCAN_AI_ENABLED, - SCAN_AI_MAX_ROWS, - SCAN_AI_MAX_TOKENS, - SCAN_AI_MODEL, - SCAN_AI_PROVIDER, - SCAN_AI_PROVIDER_LABEL, - SCAN_AI_TIMEOUT_SEC, - SCAN_CITY_AI_MAX_TOKENS, - SCAN_CITY_AI_MODEL, - SCAN_CITY_AI_RETRY_ON_STREAM_PARSE_ERROR, - SCAN_CITY_AI_STREAM_MAX_TOKENS, - SCAN_CITY_AI_TIMEOUT_SEC, SCAN_TERMINAL_BUILD_TIMEOUT_SEC, SCAN_TERMINAL_MAX_WORKERS, SCAN_TERMINAL_PAYLOAD_TTL_SEC, ) from src.data_collection.city_registry import ALIASES -from web.scan_city_ai_fallback import ( - _build_city_ai_fallback, - _complete_city_ai_payload, -) -from web.scan_city_ai_helpers import ( - _extract_ai_json_object, - _extract_provider_stream_delta, - _provider_response_meta, - _safe_float, -) -from web.scan_city_ai_prompt import ( - SCAN_CITY_AI_PROMPT_VERSION, - build_city_ai_stream_request, -) -from web.scan_city_ai_provider import ( - _call_deepseek_city_ai as _call_deepseek_city_ai_provider, -) from web.scan_terminal_cache import ( clear_scan_terminal_refreshing, - get_cached_scan_ai_result, get_cached_scan_terminal_payload, get_scan_terminal_cache_entry, mark_scan_terminal_refreshing, - set_cached_scan_ai_result, set_cached_scan_terminal_payload, set_scan_terminal_failure_state, ) from web.scan_terminal_city_row import _scan_city_terminal_rows -from web.scan_terminal_ai_compact import ( - _build_metar_decision_context, - _city_observation_anchor, - _compact_hourly_context, - _compact_probability_context, - _compact_taf_context, - build_scan_ai_prompt, -) -from web.scan_terminal_ai_merge import _normalize_ai_items, merge_scan_ai_result from web.scan_terminal_filters import ( normalize_scan_terminal_filters as _normalize_scan_terminal_filters, ) @@ -117,991 +68,6 @@ def _start_scan_terminal_background_refresh(filters: Dict[str, Any]) -> bool: return True -def _call_deepseek_scan_ai(ai_input: Dict[str, Any]) -> Dict[str, Any]: - api_key = _scan_ai_api_key() - if not api_key: - raise RuntimeError(f"{SCAN_AI_API_KEY_ENV_HINT} is not configured") - - system_prompt = ( - "你是 PolyWeather 的付费 V4-Pro 城市最高温预测员。你只能基于用户提供的 JSON 快照做判断，" - "不得编造城市、价格、概率、盘口或天气数据。输入已经按城市分组，每城包含 DEB、" - "多个天气模型预测值 model_cluster.sources、METAR 实测序列、机场原始报文和候选合约。" - "你的首要任务不是分析套利，也不是推荐 BUY YES/NO，而是预测该城市今日最终最高温是多少。" - "必须输出城市级最高温点估计、置信区间、置信度、峰值窗口状态、机场报文解读和一句预测理由。" - "最高温预测必须直接参考该城市全部 model_cluster.sources、DEB、峰值窗口和 METAR/机场报文。" - "如果天气模型之间分歧大，必须放宽置信区间并降低 confidence；如果 METAR 与模型路径冲突，必须解释修正方向。" - "必须先判断 peak_window_label、minutes_until_peak_start/end 和 window_phase：峰值窗口尚未到来时，" - "不能因为 METAR 暂未触达目标温度就下最终结论，只能说明仍需峰值窗口验证；" - "必须检查 metar_context 的 today_obs/recent_obs、max_temp、last_temp、trend_delta、" - "airport_raw_metar、airport_wx_desc、airport_cloud_desc、airport_wind_* 和 stale 状态；" - "合约只作为下游映射：可以为每个候选 row_id 给出 forecast_match（core/edge/outside/watch）和一句原因，" - "但不要输出交易建议，不要使用套利、仓位、edge 或 Kelly 语言。必须输出 JSON object。" - ) - model_snapshot = dict(ai_input) - model_snapshot.pop("_polyweather_input_meta", None) - user_payload = { - "task": ( - "Return strict JSON only with: summary_zh, summary_en, city_forecasts, contract_notes. " - "city_forecasts items require city, predicted_max, range_low, range_high, unit, confidence, " - "peak_window_zh, peak_window_en, metar_read_zh, metar_read_en, reasoning_zh, reasoning_en, model_cluster_note. " - "contract_notes items are optional and require row_id, forecast_match, reason_zh, reason_en; " - "forecast_match must be one of core, edge, outside, watch. " - "Focus on final max temperature prediction; do not output recommendations/vetoed/downgraded unless needed for backward compatibility. " - "Keep every city forecast concise: one sentence for METAR read and one sentence for reasoning." - ), - "snapshot": model_snapshot, - } - timeout = httpx.Timeout( - timeout=float(SCAN_AI_TIMEOUT_SEC), - connect=min(8.0, float(SCAN_AI_TIMEOUT_SEC)), - read=float(SCAN_AI_TIMEOUT_SEC), - write=10.0, - pool=5.0, - ) - with httpx.Client(timeout=timeout) as client: - response = client.post( - f"{SCAN_AI_BASE_URL}/chat/completions", - headers={ - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - }, - json={ - "model": SCAN_CITY_AI_MODEL, - "temperature": 0.1, - "max_tokens": SCAN_AI_MAX_TOKENS, - "response_format": {"type": "json_object"}, - "messages": [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": json.dumps(user_payload, ensure_ascii=False), - }, - ], - }, - ) - response.raise_for_status() - data = response.json() - content = ( - ((data.get("choices") or [{}])[0].get("message") or {}).get("content") - if isinstance(data, dict) - else None - ) - parsed = _extract_ai_json_object(str(content or "")) - if isinstance(data, dict): - parsed["_polyweather_meta"] = { - "usage": data.get("usage"), - "finish_reason": ((data.get("choices") or [{}])[0] or {}).get( - "finish_reason" - ), - } - return parsed - - -def _build_city_ai_prompt(data: Dict[str, Any]) -> Dict[str, Any]: - local_date = str(data.get("local_date") or "").strip() - multi_model_daily = ( - data.get("multi_model_daily") - if isinstance(data.get("multi_model_daily"), dict) - else {} - ) - daily_entry = ( - multi_model_daily.get(local_date) if isinstance(multi_model_daily, dict) else {} - ) - if not isinstance(daily_entry, dict): - daily_entry = {} - daily_models = ( - daily_entry.get("models") - if isinstance(daily_entry.get("models"), dict) - else None - ) - models = daily_models or ( - data.get("multi_model") if isinstance(data.get("multi_model"), dict) else {} - ) - model_values = [_safe_float(value) for value in (models or {}).values()] - model_values = [value for value in model_values if value is not None] - metar_context = _build_metar_decision_context(data) - observation_anchor = _city_observation_anchor(data) - current = data.get("current") if isinstance(data.get("current"), dict) else {} - airport_current = ( - data.get("airport_current") - if isinstance(data.get("airport_current"), dict) - else {} - ) - airport_primary = ( - data.get("airport_primary") - if isinstance(data.get("airport_primary"), dict) - else {} - ) - risk = data.get("risk") if isinstance(data.get("risk"), dict) else {} - - return { - "schema_version": "single_city_forecast_v2", - "prompt_version": SCAN_CITY_AI_PROMPT_VERSION, - "task": "predict_city_daily_high_and_read_observation", - "city": data.get("name"), - "city_display_name": data.get("display_name") or data.get("name"), - "local_date": local_date, - "local_time": data.get("local_time"), - "temp_symbol": data.get("temp_symbol"), - "timezone_offset_seconds": data.get("utc_offset_seconds"), - "observation_anchor": observation_anchor, - "settlement_station": data.get("settlement_station") or {}, - "current": { - "temp": current.get("temp"), - "max_so_far": current.get("max_so_far"), - "max_temp_time": current.get("max_temp_time"), - "obs_time": current.get("obs_time"), - "station_code": current.get("station_code"), - "station_name": current.get("station_name"), - "wind_speed_kt": current.get("wind_speed_kt"), - "wind_dir": current.get("wind_dir"), - "humidity": current.get("humidity"), - "pressure_hpa": current.get("pressure_hpa"), - "observation_source": current.get("settlement_source"), - }, - "airport": { - "name": risk.get("airport") - or airport_current.get("station_label") - or airport_primary.get("station_label"), - "icao": risk.get("icao") - or airport_current.get("station_code") - or airport_primary.get("station_code"), - "distance_km": risk.get("distance_km"), - }, - "model_cluster": { - "sources": [ - *( - [ - { - "model": "DEB (fusion)", - "value": ( - (daily_entry.get("deb") or {}).get("prediction") - if isinstance(daily_entry.get("deb"), dict) - else None - ) - or ( - (data.get("deb") or {}).get("prediction") - if isinstance(data.get("deb"), dict) - else None - ), - } - ] - if ( - ( - (daily_entry.get("deb") or {}).get("prediction") - if isinstance(daily_entry.get("deb"), dict) - else None - ) - or ( - (data.get("deb") or {}).get("prediction") - if isinstance(data.get("deb"), dict) - else None - ) - ) - is not None - else [] - ), - *[ - {"model": str(name), "value": value} - for name, value in (models or {}).items() - if _safe_float(value) is not None - ], - ], - "model_count": len(model_values) - + ( - 1 - if ( - ( - (daily_entry.get("deb") or {}).get("prediction") - if isinstance(daily_entry.get("deb"), dict) - else None - ) - or ( - (data.get("deb") or {}).get("prediction") - if isinstance(data.get("deb"), dict) - else None - ) - ) - is not None - else 0 - ), - "min": min(model_values) if model_values else None, - "max": max(model_values) if model_values else None, - "spread": (max(model_values) - min(model_values)) - if len(model_values) >= 2 - else None, - }, - "peak": data.get("peak") or {}, - "metar_context": metar_context, - "airport_current": { - "temp": airport_current.get("temp"), - "obs_time": airport_current.get("obs_time"), - "report_time": airport_current.get("report_time"), - "receipt_time": airport_current.get("receipt_time"), - "wind_speed_kt": airport_current.get("wind_speed_kt"), - "wind_dir": airport_current.get("wind_dir"), - "humidity": airport_current.get("humidity"), - "cloud_desc": airport_current.get("cloud_desc"), - "visibility_mi": airport_current.get("visibility_mi"), - "wx_desc": airport_current.get("wx_desc"), - "raw_metar": airport_current.get("raw_metar"), - "station_code": airport_current.get("station_code"), - "station_label": airport_current.get("station_label"), - }, - "taf": _compact_taf_context(data.get("taf")), - "probability": _compact_probability_context( - data.get("probabilities") - if isinstance(data.get("probabilities"), dict) - else None, - data.get("deb") if isinstance(data.get("deb"), dict) else None, - data.get("temp_symbol"), - ), - "hourly": _compact_hourly_context(data.get("hourly")), - } - - -def _call_deepseek_city_ai( - ai_input: Dict[str, Any], *, locale: str = "zh-CN" -) -> Dict[str, Any]: - return _call_deepseek_city_ai_provider( - ai_input, - locale=locale, - api_key=_scan_ai_api_key(), - base_url=SCAN_AI_BASE_URL, - model=SCAN_CITY_AI_MODEL, - max_tokens=SCAN_CITY_AI_MAX_TOKENS, - timeout_sec=SCAN_CITY_AI_TIMEOUT_SEC, - ) - - -def _scan_city_ai_cache_key(ai_input: Dict[str, Any]) -> str: - observation_anchor = ( - ai_input.get("observation_anchor") - if isinstance(ai_input.get("observation_anchor"), dict) - else {} - ) - is_airport_metar = observation_anchor.get("is_airport_metar") is not False - airport_current = ( - ai_input.get("airport_current") - if isinstance(ai_input.get("airport_current"), dict) - else {} - ) - metar_context = ( - ai_input.get("metar_context") - if isinstance(ai_input.get("metar_context"), dict) - else {} - ) - key_payload = { - "prompt_version": SCAN_CITY_AI_PROMPT_VERSION, - "city": ai_input.get("city"), - "local_date": ai_input.get("local_date"), - "station": observation_anchor.get("station_code"), - "raw_metar": airport_current.get("raw_metar") if is_airport_metar else None, - "obs_time": airport_current.get("obs_time") - or metar_context.get("last_observation_time"), - "stale_for_today": metar_context.get("stale_for_today"), - } - raw = json.dumps(key_payload, sort_keys=True, ensure_ascii=False, default=str) - return "city-ai:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() - - -def _quick_metar_cache_key(data: Dict[str, Any]) -> str: - airport_current = ( - data.get("airport_current") - if isinstance(data.get("airport_current"), dict) - else {} - ) - current = data.get("current") if isinstance(data.get("current"), dict) else {} - observation_anchor = ( - data.get("observation_anchor") - if isinstance(data.get("observation_anchor"), dict) - else {} - ) - raw_metar = airport_current.get("raw_metar") or current.get("raw_metar") - obs_time = airport_current.get("obs_time") or current.get("obs_time") - if raw_metar and obs_time: - finger = { - "city": data.get("name"), - "raw_metar": raw_metar, - "obs_time": obs_time, - "station": observation_anchor.get("station_code"), - "prompt_version": SCAN_CITY_AI_PROMPT_VERSION, - } - raw = json.dumps(finger, sort_keys=True, ensure_ascii=False, default=str) - return "city-ai:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() - return "" - - -def _sse_event(event: str, payload: Dict[str, Any]) -> str: - return ( - f"event: {event}\n" - f"data: {json.dumps(payload, ensure_ascii=False, default=str)}\n\n" - ) - - -def _build_city_ai_stream_request( - ai_input: Dict[str, Any], - *, - locale: str, -) -> Dict[str, Any]: - return build_city_ai_stream_request( - ai_input, - locale=locale, - model=SCAN_CITY_AI_MODEL, - max_tokens=SCAN_CITY_AI_STREAM_MAX_TOKENS, - ) - - -def _cache_city_ai_payload( - cache_key: str, - *, - data: Dict[str, Any], - generated_at: str, - ai_raw: Dict[str, Any], - quick_key: str = "", -) -> None: - entry = { - "expires_at": time.time() + SCAN_AI_CACHE_TTL_SEC, - "generated_at": generated_at, - "city": data.get("name"), - "city_display_name": data.get("display_name"), - "payload": ai_raw, - } - with _SCAN_CITY_AI_CACHE_LOCK: - _SCAN_CITY_AI_CACHE[cache_key] = entry - if quick_key and quick_key != cache_key: - _SCAN_CITY_AI_CACHE[quick_key] = entry - - -def _is_city_ai_fallback(ai_raw: Any) -> bool: - if not isinstance(ai_raw, dict): - return True - meta = ai_raw.get("_polyweather_meta") - return bool(isinstance(meta, dict) and meta.get("fallback")) - - -def _build_city_ai_result_payload( - *, - data: Dict[str, Any], - generated_at: str, - started_at: float, - ai_raw: Dict[str, Any], - cached: bool = False, - degraded: bool = False, - reason: Optional[str] = None, - reason_zh: Optional[str] = None, - reason_en: Optional[str] = None, -) -> Dict[str, Any]: - payload: Dict[str, Any] = { - "status": "ready", - "cached": cached, - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name"), - "city_display_name": data.get("display_name"), - "generated_at": generated_at, - "duration_ms": int((time.time() - started_at) * 1000), - "city_forecast": ai_raw, - } - if degraded: - payload["degraded"] = True - if reason: - payload["reason"] = reason - if reason_zh: - payload["reason_zh"] = reason_zh - if reason_en: - payload["reason_en"] = reason_en - return payload - - -def stream_scan_city_ai_forecast_payload( - city: str, - *, - force_refresh: bool = False, - locale: str = "zh-CN", -) -> Iterator[str]: - started_at = time.time() - city_name = _normalize_city_key(city) - normalized_locale = _normalize_locale(locale) - if not city_name: - yield _sse_event("final", {"status": "failed", "reason": "city is required"}) - return - if city_name not in CITIES: - reason_en = f"Unknown city: {city_name}" - reason_zh = f"未知城市：{city_name}" - yield _sse_event( - "final", - { - "status": "failed", - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": city_name, - "city_display_name": str(city or "").strip() or city_name, - "reason": reason_en if normalized_locale == "en-US" else reason_zh, - "reason_en": reason_en, - "reason_zh": reason_zh, - }, - ) - return - - yield _sse_event( - "progress", - { - "stage": "loading_city", - "message_zh": "正在读取城市实况、模型和最新观测…", - "message_en": "Loading city observations, model cluster and latest observation…", - }, - ) - data = _analyze( - city_name, - force_refresh=False, - include_llm_commentary=False, - detail_mode="full", - ) - ai_input = _build_city_ai_prompt(data) - cache_key = _scan_city_ai_cache_key(ai_input) - quick_key = _quick_metar_cache_key(data) - if not force_refresh: - with _SCAN_CITY_AI_CACHE_LOCK: - cached = _SCAN_CITY_AI_CACHE.get(cache_key) or ( - _SCAN_CITY_AI_CACHE.get(quick_key) if quick_key else None - ) - if cached and cached.get("expires_at", 0) >= time.time(): - yield _sse_event( - "final", - { - "status": "ready", - "cached": True, - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": cached.get("city") or city_name, - "city_display_name": cached.get("city_display_name") - or city_name, - "generated_at": cached.get("generated_at"), - "duration_ms": 0, - "city_forecast": cached.get("payload"), - }, - ) - return - preview_raw = _build_city_ai_fallback( - ai_input, - locale=normalized_locale, - reason="stream preview", - ) - observation_anchor = ( - ai_input.get("observation_anchor") - if isinstance(ai_input.get("observation_anchor"), dict) - else {} - ) - is_airport_metar = observation_anchor.get("is_airport_metar") is not False - calling_message_zh = ( - f"{SCAN_AI_PROVIDER_LABEL} 正在快速增强机场报文解读…" - if is_airport_metar - else f"{SCAN_AI_PROVIDER_LABEL} 正在快速增强香港天文台观测解读…" - ) - calling_message_en = ( - f"{SCAN_AI_PROVIDER_LABEL} is adding a fast airport-bulletin enhancement…" - if is_airport_metar - else f"{SCAN_AI_PROVIDER_LABEL} is adding a fast HKO observation enhancement…" - ) - yield _sse_event( - "preview", - { - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "metar_read_zh": preview_raw.get("metar_read_zh"), - "metar_read_en": preview_raw.get("metar_read_en"), - "final_judgment_zh": preview_raw.get("final_judgment_zh"), - "final_judgment_en": preview_raw.get("final_judgment_en"), - "model_cluster_note_zh": preview_raw.get("model_cluster_note_zh"), - "model_cluster_note_en": preview_raw.get("model_cluster_note_en"), - "predicted_max": preview_raw.get("predicted_max"), - "range_low": preview_raw.get("range_low"), - "range_high": preview_raw.get("range_high"), - "confidence": preview_raw.get("confidence"), - "unit": preview_raw.get("unit"), - }, - ) - yield _sse_event( - "progress", - { - "stage": "calling_ai", - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "message_zh": calling_message_zh, - "message_en": calling_message_en, - }, - ) - - if not SCAN_AI_ENABLED: - yield _sse_event( - "final", - { - "status": "disabled", - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "reason": "POLYWEATHER_SCAN_AI_ENABLED is not enabled", - }, - ) - return - if not _scan_ai_api_key(): - yield _sse_event( - "final", - { - "status": "missing_key", - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "reason": f"{SCAN_AI_API_KEY_ENV_HINT} is not configured", - }, - ) - return - - request_json = _build_city_ai_stream_request(ai_input, locale=normalized_locale) - timeout = httpx.Timeout( - timeout=float(SCAN_CITY_AI_TIMEOUT_SEC), - connect=min(8.0, float(SCAN_CITY_AI_TIMEOUT_SEC)), - read=float(SCAN_CITY_AI_TIMEOUT_SEC), - write=10.0, - pool=5.0, - ) - headers = { - "Authorization": f"Bearer {_scan_ai_api_key()}", - "Content-Type": "application/json", - } - accumulated = "" - last_meta: Dict[str, Any] = {} - try: - logger.info( - "scan city AI stream request city={} locale={} input_bytes={} max_tokens={} timeout_sec={}", - ai_input.get("city"), - normalized_locale, - len( - json.dumps(request_json, ensure_ascii=False, default=str).encode( - "utf-8" - ) - ), - request_json.get("max_tokens"), - SCAN_CITY_AI_TIMEOUT_SEC, - ) - with httpx.Client(timeout=timeout) as client: - with client.stream( - "POST", - f"{SCAN_AI_BASE_URL}/chat/completions", - headers=headers, - json=request_json, - ) as response: - response.raise_for_status() - for line in response.iter_lines(): - text = str(line or "").strip() - if not text or not text.startswith("data:"): - continue - payload_text = text[5:].strip() - if payload_text == "[DONE]": - break - try: - chunk = json.loads(payload_text) - except Exception: - continue - last_meta = _provider_response_meta(chunk) or last_meta - delta = _extract_provider_stream_delta(chunk) - if delta: - accumulated += delta - yield _sse_event( - "delta", - { - "content": delta, - "raw_length": len(accumulated), - }, - ) - degraded = False - degraded_reason: Optional[str] = None - try: - ai_raw = _extract_ai_json_object(accumulated) - if isinstance(ai_raw, dict): - ai_raw["_polyweather_meta"] = { - **last_meta, - "streamed": True, - } - ai_raw = _complete_city_ai_payload( - ai_raw, - ai_input, - locale=normalized_locale, - ) - except Exception as exc: - retry_reason = str(exc) - if SCAN_CITY_AI_RETRY_ON_STREAM_PARSE_ERROR: - yield _sse_event( - "progress", - { - "stage": "retry_non_stream", - "message_zh": "流式内容为空或 JSON 不完整，正在改用非流式严格 JSON 重试…", - "message_en": "Stream content was empty or incomplete JSON; retrying with a strict non-stream request…", - "raw_length": len(accumulated), - "reason": retry_reason, - }, - ) - try: - ai_raw = _call_deepseek_city_ai(ai_input, locale=normalized_locale) - if isinstance(ai_raw, dict): - meta = ai_raw.get("_polyweather_meta") - if not isinstance(meta, dict): - meta = {} - ai_raw["_polyweather_meta"] = { - **meta, - "streamed": False, - "stream_retry_non_stream": True, - "stream_retry_reason": retry_reason, - "stream_raw_length": len(accumulated), - } - if _is_city_ai_fallback(ai_raw): - degraded = True - degraded_reason = retry_reason - except Exception as retry_exc: - degraded = True - degraded_reason = str(retry_exc) - ai_raw = _build_city_ai_fallback( - ai_input, - locale=normalized_locale, - reason=degraded_reason or retry_reason, - raw_content=accumulated, - ) - else: - degraded = True - degraded_reason = retry_reason - ai_raw = _build_city_ai_fallback( - ai_input, - locale=normalized_locale, - reason=retry_reason, - raw_content=accumulated, - ) - generated_at = datetime.utcnow().isoformat() + "Z" - if not _is_city_ai_fallback(ai_raw): - _cache_city_ai_payload( - cache_key, - data=data, - generated_at=generated_at, - ai_raw=ai_raw, - quick_key=quick_key, - ) - yield _sse_event( - "final", - _build_city_ai_result_payload( - data=data, - generated_at=generated_at, - started_at=started_at, - ai_raw=ai_raw, - degraded=degraded, - reason=degraded_reason, - reason_en=degraded_reason, - reason_zh=degraded_reason, - ), - ) - except httpx.TimeoutException as exc: - duration_ms = int((time.time() - started_at) * 1000) - reason_en = f"{SCAN_AI_PROVIDER_LABEL} city AI timed out after {SCAN_CITY_AI_TIMEOUT_SEC}s" - reason_zh = ( - f"{SCAN_AI_PROVIDER_LABEL} 城市 AI 在 {SCAN_CITY_AI_TIMEOUT_SEC} 秒内未返回" - ) - logger.warning( - "scan city AI stream timeout fallback city={} duration_ms={} model={} error={}", - data.get("name") or city_name, - duration_ms, - SCAN_CITY_AI_MODEL, - exc, - ) - ai_raw = _build_city_ai_fallback( - ai_input, - locale=normalized_locale, - reason=reason_en if normalized_locale == "en-US" else reason_zh, - raw_content=accumulated, - ) - generated_at = datetime.utcnow().isoformat() + "Z" - yield _sse_event( - "final", - _build_city_ai_result_payload( - data=data, - generated_at=generated_at, - started_at=started_at, - ai_raw=ai_raw, - degraded=True, - reason=reason_en if normalized_locale == "en-US" else reason_zh, - reason_en=reason_en, - reason_zh=reason_zh, - ), - ) - except Exception as exc: - reason = str(exc) - logger.warning( - "scan city AI stream failed city={} model={} error={}", - data.get("name") or city_name, - SCAN_CITY_AI_MODEL, - reason, - ) - yield _sse_event( - "final", - { - "status": "failed", - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "duration_ms": int((time.time() - started_at) * 1000), - "reason": reason, - "reason_en": reason, - "reason_zh": reason, - "raw_reason": reason, - }, - ) - - -def build_scan_city_ai_forecast_payload( - city: str, - *, - force_refresh: bool = False, - locale: str = "zh-CN", -) -> Dict[str, Any]: - started_at = time.time() - city_name = _normalize_city_key(city) - normalized_locale = _normalize_locale(locale) - if not city_name: - return {"status": "failed", "reason": "city is required"} - if city_name not in CITIES: - return { - "status": "failed", - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": city_name, - "city_display_name": str(city or "").strip() or city_name, - "reason": ( - f"Unknown city: {city_name}" - if normalized_locale == "en-US" - else f"未知城市：{city_name}" - ), - "reason_en": f"Unknown city: {city_name}", - "reason_zh": f"未知城市：{city_name}", - } - logger.info( - "scan city AI forecast requested city={} force_refresh={} locale={} model={}", - city_name, - force_refresh, - normalized_locale, - SCAN_CITY_AI_MODEL, - ) - data = _analyze( - city_name, - force_refresh=False, - include_llm_commentary=False, - detail_mode="full", - ) - ai_input = _build_city_ai_prompt(data) - cache_key = _scan_city_ai_cache_key(ai_input) - quick_key = _quick_metar_cache_key(data) - if not force_refresh: - with _SCAN_CITY_AI_CACHE_LOCK: - cached = _SCAN_CITY_AI_CACHE.get(cache_key) or ( - _SCAN_CITY_AI_CACHE.get(quick_key) if quick_key else None - ) - if cached and cached.get("expires_at", 0) >= time.time(): - logger.info( - "scan city AI forecast cache hit city={} model={}", - cached.get("city") or city_name, - SCAN_CITY_AI_MODEL, - ) - return { - "status": "ready", - "cached": True, - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": cached.get("city") or city_name, - "city_display_name": cached.get("city_display_name") or city_name, - "generated_at": cached.get("generated_at"), - "duration_ms": 0, - "city_forecast": cached.get("payload"), - } - - if not SCAN_AI_ENABLED: - logger.warning( - "scan city AI forecast disabled city={} model={}", - data.get("name") or city_name, - SCAN_CITY_AI_MODEL, - ) - return { - "status": "disabled", - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "reason": "POLYWEATHER_SCAN_AI_ENABLED is not enabled", - } - if not _scan_ai_api_key(): - logger.warning( - "scan city AI forecast missing provider key city={} model={}", - data.get("name") or city_name, - SCAN_CITY_AI_MODEL, - ) - return { - "status": "missing_key", - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "reason": f"{SCAN_AI_API_KEY_ENV_HINT} is not configured", - } - - try: - logger.info( - "scan city AI forecast calling provider city={} station={} model={} raw_metar_present={}", - data.get("name") or city_name, - ( - (ai_input.get("airport") or {}).get("icao") - if isinstance(ai_input.get("airport"), dict) - else None - ), - SCAN_CITY_AI_MODEL, - bool( - (ai_input.get("airport_current") or {}).get("raw_metar") - if isinstance(ai_input.get("airport_current"), dict) - else False - ), - ) - ai_raw = _call_deepseek_city_ai(ai_input, locale=normalized_locale) - except httpx.TimeoutException as exc: - duration_ms = int((time.time() - started_at) * 1000) - reason_en = f"{SCAN_AI_PROVIDER_LABEL} city AI timed out after {SCAN_CITY_AI_TIMEOUT_SEC}s" - reason_zh = ( - f"{SCAN_AI_PROVIDER_LABEL} 城市 AI 在 {SCAN_CITY_AI_TIMEOUT_SEC} 秒内未返回" - ) - ai_raw = _build_city_ai_fallback( - ai_input, - locale=normalized_locale, - reason=reason_en if normalized_locale == "en-US" else reason_zh, - ) - generated_at = datetime.utcnow().isoformat() + "Z" - logger.warning( - "scan city AI forecast timeout fallback city={} duration_ms={} model={} error={}", - data.get("name") or city_name, - duration_ms, - SCAN_CITY_AI_MODEL, - exc, - ) - return { - "status": "ready", - "degraded": True, - "cached": False, - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "generated_at": generated_at, - "duration_ms": duration_ms, - "reason": reason_en if normalized_locale == "en-US" else reason_zh, - "reason_en": reason_en, - "reason_zh": reason_zh, - "city_forecast": ai_raw, - } - except Exception as exc: - duration_ms = int((time.time() - started_at) * 1000) - raw_reason = str(exc) - empty_ai_content = raw_reason.strip().lower() == "empty ai content" - reason_en = ( - f"{SCAN_AI_PROVIDER_LABEL} city AI returned no usable text. Retry the city analysis." - if empty_ai_content - else raw_reason - ) - reason_zh = ( - f"{SCAN_AI_PROVIDER_LABEL} 城市 AI 没有返回有效正文，请刷新重试。" - if empty_ai_content - else raw_reason - ) - logger.warning( - "scan city AI forecast failed city={} duration_ms={} model={} error={}", - data.get("name") or city_name, - duration_ms, - SCAN_CITY_AI_MODEL, - raw_reason, - ) - ai_raw = _build_city_ai_fallback( - ai_input, - locale=normalized_locale, - reason=reason_en if normalized_locale == "en-US" else reason_zh, - ) - generated_at = datetime.utcnow().isoformat() + "Z" - return { - "status": "ready", - "degraded": True, - "cached": False, - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "generated_at": generated_at, - "duration_ms": duration_ms, - "reason": reason_en if normalized_locale == "en-US" else reason_zh, - "reason_en": reason_en, - "reason_zh": reason_zh, - "raw_reason": raw_reason, - "city_forecast": ai_raw, - } - generated_at = datetime.utcnow().isoformat() + "Z" - _cache_city_ai_payload( - cache_key, - data=data, - generated_at=generated_at, - ai_raw=ai_raw, - quick_key=quick_key, - ) - logger.info( - "scan city AI forecast complete city={} duration_ms={} model={} confidence={}", - data.get("name") or city_name, - int((time.time() - started_at) * 1000), - SCAN_CITY_AI_MODEL, - ai_raw.get("confidence") if isinstance(ai_raw, dict) else None, - ) - return { - "status": "ready", - "cached": False, - "model": SCAN_CITY_AI_MODEL, - "provider": SCAN_AI_PROVIDER, - "city": data.get("name") or city_name, - "city_display_name": data.get("display_name") or city_name, - "generated_at": generated_at, - "duration_ms": int((time.time() - started_at) * 1000), - "city_forecast": ai_raw, - } - - -def _build_scan_ai_unavailable_payload( - payload: Dict[str, Any], - *, - status: str, - reason: str, - duration_ms: Optional[int] = None, -) -> Dict[str, Any]: - return { - **payload, - "ai_scan": { - "status": status, - "stage": "fallback", - "model": SCAN_AI_MODEL, - "cached": False, - "generated_at": datetime.utcnow().isoformat() + "Z", - "snapshot_id": payload.get("snapshot_id"), - "input_rows": len(payload.get("rows") or []), - "sent_rows": min(len(payload.get("rows") or []), SCAN_AI_MAX_ROWS), - "duration_ms": duration_ms, - "timeout_sec": SCAN_AI_TIMEOUT_SEC, - "cache_ttl_sec": SCAN_AI_CACHE_TTL_SEC, - "provider": SCAN_AI_PROVIDER, - "base_url": SCAN_AI_BASE_URL, - "reason": reason, - }, - } - - def _build_scan_terminal_payload_uncached( filters: Dict[str, Any], *, @@ -1294,157 +260,6 @@ def build_scan_terminal_payload( return _build_scan_terminal_payload_uncached(filters, force_refresh=force_refresh) -def build_scan_terminal_ai_payload( - raw_filters: Optional[Dict[str, Any]] = None, - *, - snapshot_id: Optional[str] = None, -) -> Dict[str, Any]: - ai_started_at = time.time() - filters = _normalize_scan_terminal_filters(raw_filters) - payload = build_scan_terminal_payload(filters, force_refresh=False) - current_snapshot_id = str(payload.get("snapshot_id") or "").strip() - requested_snapshot_id = str(snapshot_id or "").strip() - if ( - requested_snapshot_id - and current_snapshot_id - and requested_snapshot_id != current_snapshot_id - ): - return _build_scan_ai_unavailable_payload( - payload, - status="snapshot_mismatch", - reason="scan snapshot changed; refresh the scan before running AI review", - ) - if not current_snapshot_id: - return _build_scan_ai_unavailable_payload( - payload, - status="no_snapshot", - reason="no scan snapshot is available for AI review", - ) - if not payload.get("rows"): - return _build_scan_ai_unavailable_payload( - payload, - status="no_rows", - reason="no candidate rows are available for AI review", - ) - if not SCAN_AI_ENABLED: - return _build_scan_ai_unavailable_payload( - payload, - status="disabled", - reason="POLYWEATHER_SCAN_AI_ENABLED is not enabled", - ) - if not _scan_ai_api_key(): - return _build_scan_ai_unavailable_payload( - payload, - status="missing_key", - reason=f"{SCAN_AI_API_KEY_ENV_HINT} is not configured", - ) - - cached = get_cached_scan_ai_result( - current_snapshot_id, - filters, - max_rows=SCAN_AI_MAX_ROWS, - model=SCAN_AI_MODEL, - ttl_sec=SCAN_AI_CACHE_TTL_SEC, - ) - if cached is not None: - logger.info( - "scan terminal AI cache hit snapshot={} rows={}", - current_snapshot_id, - len(payload.get("rows") or []), - ) - return merge_scan_ai_result( - payload, - cached, - model=SCAN_AI_MODEL, - max_rows=SCAN_AI_MAX_ROWS, - timeout_sec=SCAN_AI_TIMEOUT_SEC, - cache_ttl_sec=SCAN_AI_CACHE_TTL_SEC, - base_url=SCAN_AI_BASE_URL, - cached=True, - provider=SCAN_AI_PROVIDER, - duration_ms=0, - input_rows=len(payload.get("rows") or []), - ) - - try: - ai_input = build_scan_ai_prompt(payload, max_rows=SCAN_AI_MAX_ROWS) - input_meta = ( - ai_input.get("_polyweather_input_meta") - if isinstance(ai_input, dict) - else {} - ) - sent_rows = int((input_meta or {}).get("sent_contracts") or 0) - sent_cities = int((input_meta or {}).get("sent_cities") or 0) - logger.info( - "scan terminal AI review start snapshot={} rows={} sent_cities={} sent_contracts={} model={}", - current_snapshot_id, - len(payload.get("rows") or []), - sent_cities, - sent_rows, - SCAN_AI_MODEL, - ) - ai_raw = _call_deepseek_scan_ai(ai_input) - ai_raw["_polyweather_input_meta"] = input_meta - set_cached_scan_ai_result( - current_snapshot_id, - filters, - ai_raw, - max_rows=SCAN_AI_MAX_ROWS, - model=SCAN_AI_MODEL, - ) - duration_ms = int((time.time() - ai_started_at) * 1000) - logger.info( - "scan terminal AI review complete snapshot={} duration_ms={} recommendations={} vetoed={} downgraded={}", - current_snapshot_id, - duration_ms, - len(_normalize_ai_items(ai_raw.get("recommendations"))), - len(_normalize_ai_items(ai_raw.get("vetoed"))), - len(_normalize_ai_items(ai_raw.get("downgraded"))), - ) - return merge_scan_ai_result( - payload, - ai_raw, - model=SCAN_AI_MODEL, - max_rows=SCAN_AI_MAX_ROWS, - timeout_sec=SCAN_AI_TIMEOUT_SEC, - cache_ttl_sec=SCAN_AI_CACHE_TTL_SEC, - base_url=SCAN_AI_BASE_URL, - cached=False, - provider=SCAN_AI_PROVIDER, - duration_ms=duration_ms, - input_rows=len(payload.get("rows") or []), - ) - except httpx.TimeoutException as exc: - duration_ms = int((time.time() - ai_started_at) * 1000) - reason = f"V4 provider timed out after {SCAN_AI_TIMEOUT_SEC}s" - logger.warning( - "scan terminal AI review timeout snapshot={} duration_ms={} error={}", - current_snapshot_id, - duration_ms, - exc, - ) - return _build_scan_ai_unavailable_payload( - payload, - status="timeout", - reason=reason, - duration_ms=duration_ms, - ) - except Exception as exc: - duration_ms = int((time.time() - ai_started_at) * 1000) - logger.warning( - "scan terminal AI review failed snapshot={} duration_ms={} error={}", - current_snapshot_id, - duration_ms, - exc, - ) - return _build_scan_ai_unavailable_payload( - payload, - status="failed", - reason=str(exc), - duration_ms=duration_ms, - ) - - _SCAN_PREWARM_STARTED = False _SCAN_PREWARM_LOCK = threading.Lock() diff --git a/web/services/city_runtime.py b/web/services/city_runtime.py index 8b73f3dc..e04212f5 100644 --- a/web/services/city_runtime.py +++ b/web/services/city_runtime.py @@ -3,7 +3,7 @@ from __future__ import annotations import os import time from datetime import datetime -from typing import Optional +from typing import Any, Dict, Iterator, Optional from fastapi import APIRouter, BackgroundTasks, HTTPException from loguru import logger @@ -20,6 +20,7 @@ from src.analysis.settlement_rounding import apply_city_settlement from src.data_collection.country_networks import get_country_network_provider # noqa: F401 - compatibility export for transitional routers from src.data_collection.city_registry import ALIASES from src.data_collection.city_time import get_city_utc_offset_seconds # noqa: F401 - compatibility export for transitional routers +from src.utils.refresh_policy import OBSERVATION_REFRESH_SEC, SCAN_ROWS_REFRESH_SEC from web.analysis_service import ( _analyze, _analyze_summary, @@ -27,12 +28,7 @@ from web.analysis_service import ( _build_city_market_scan_payload, _build_city_summary_payload, ) -from web.scan_terminal_service import ( - build_scan_city_ai_forecast_payload, # noqa: F401 - compatibility export for tests and transitional routers - build_scan_terminal_ai_payload, # noqa: F401 - compatibility export for tests and transitional routers - build_scan_terminal_payload, # noqa: F401 - compatibility export for tests and transitional routers - stream_scan_city_ai_forecast_payload, # noqa: F401 - compatibility export for tests and transitional routers -) +from web.scan_terminal_service import build_scan_terminal_payload # noqa: F401 - compatibility export for tests and transitional routers from web.core import ( CITIES, CITY_REGISTRY, # noqa: F401 - compatibility export for tests and transitional routers @@ -62,6 +58,50 @@ from web.core import ( router = APIRouter() _CACHE_DB = DBManager() + +def build_scan_terminal_ai_payload( + raw_filters: Optional[Dict[str, Any]] = None, + *, + snapshot_id: Optional[str] = None, +) -> Dict[str, Any]: + return { + "available": False, + "status": "disabled", + "reason": "scan AI has been removed", + "snapshot_id": snapshot_id, + "rows": [], + } + + +def build_scan_city_ai_forecast_payload( + city: str, + *, + force_refresh: bool = False, + locale: str = "zh-CN", +) -> Dict[str, Any]: + return { + "available": False, + "status": "disabled", + "reason": "city AI has been removed", + "city": city, + "locale": locale, + "force_refresh": force_refresh, + } + + +def stream_scan_city_ai_forecast_payload( + city: str, + *, + force_refresh: bool = False, + locale: str = "zh-CN", +) -> Iterator[str]: + payload = build_scan_city_ai_forecast_payload( + city, + force_refresh=force_refresh, + locale=locale, + ) + yield f"data: {payload}\n\n" + _DEB_RECENT_LOOKBACK = 7 _DEB_RECENT_MIN_SAMPLES = 3 _daily_record_repo = DailyRecordRepository() @@ -138,11 +178,11 @@ US_CORE_CITIES = [ "atlanta", "seattle", ] -CITY_SUMMARY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_SUMMARY_CACHE_TTL_SEC", "1800"))) -CITY_PANEL_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_PANEL_CACHE_TTL_SEC", "1800"))) -CITY_NEARBY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_NEARBY_CACHE_TTL_SEC", "1800"))) -CITY_MARKET_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_MARKET_CACHE_TTL_SEC", "1800"))) -CITY_FULL_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_FULL_CACHE_TTL_SEC", "1800"))) +CITY_SUMMARY_CACHE_TTL_SEC = min(SCAN_ROWS_REFRESH_SEC, max(30, int(os.getenv("POLYWEATHER_CITY_SUMMARY_CACHE_TTL_SEC", str(SCAN_ROWS_REFRESH_SEC))))) +CITY_PANEL_CACHE_TTL_SEC = min(SCAN_ROWS_REFRESH_SEC, max(30, int(os.getenv("POLYWEATHER_CITY_PANEL_CACHE_TTL_SEC", str(SCAN_ROWS_REFRESH_SEC))))) +CITY_NEARBY_CACHE_TTL_SEC = min(SCAN_ROWS_REFRESH_SEC, max(30, int(os.getenv("POLYWEATHER_CITY_NEARBY_CACHE_TTL_SEC", str(SCAN_ROWS_REFRESH_SEC))))) +CITY_MARKET_CACHE_TTL_SEC = min(SCAN_ROWS_REFRESH_SEC, max(30, int(os.getenv("POLYWEATHER_CITY_MARKET_CACHE_TTL_SEC", str(SCAN_ROWS_REFRESH_SEC))))) +CITY_FULL_CACHE_TTL_SEC = min(OBSERVATION_REFRESH_SEC, max(30, int(os.getenv("POLYWEATHER_CITY_FULL_CACHE_TTL_SEC", str(OBSERVATION_REFRESH_SEC))))) MARKET_SCAN_PAYLOAD_TTL_SEC = max( 5, int(os.getenv("POLYWEATHER_MARKET_SCAN_PAYLOAD_TTL_SEC", "30")), @@ -321,7 +361,7 @@ def _refresh_city_market_cache(city: str, force_refresh: bool = False) -> dict: def _refresh_city_full_cache(city: str, force_refresh: bool = False) -> dict: - payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=True, detail_mode="full") + payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=False, detail_mode="full") _CACHE_DB.set_city_cache( "full", city, diff --git a/web/services/market_overview_api.py b/web/services/market_overview_api.py index 9b14f2ba..2d07d713 100644 --- a/web/services/market_overview_api.py +++ b/web/services/market_overview_api.py @@ -1,4 +1,4 @@ -"""Market overview — AI summary of all scan terminal rows, cached 10 min.""" +"""Deterministic market overview for scan terminal rows, cached 10 minutes.""" from __future__ import annotations @@ -7,108 +7,136 @@ import json import threading import time from datetime import datetime -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional -from loguru import logger - -from web.scan_city_ai_helpers import _safe_float -from web.scan_terminal_service import ( - SCAN_AI_BASE_URL, - SCAN_CITY_AI_MODEL, - SCAN_CITY_AI_TIMEOUT_SEC, - _scan_ai_api_key, -) +from src.utils.refresh_policy import MARKET_OVERVIEW_TTL_SEC _OVERVIEW_CACHE: Dict[str, Dict[str, Any]] = {} _OVERVIEW_CACHE_LOCK = threading.Lock() -_OVERVIEW_MAX_TOKENS = 600 -_OVERVIEW_CACHE_TTL_SEC = 600 +OVERVIEW_CACHE_TTL_SEC = MARKET_OVERVIEW_TTL_SEC -def _build_overview_ai_request( - rows: List[Dict[str, Any]], - locale: str, -) -> Dict[str, Any]: - cities = [] - for row in rows: - if not isinstance(row, dict): - continue - city = row.get("city") or row.get("name") or "" - if not city: - continue - model_cluster = row.get("model_cluster") if isinstance(row.get("model_cluster"), dict) else {} - sources = model_cluster.get("sources") if isinstance(model_cluster.get("sources"), list) else [] - values = [ - _safe_float(s.get("value")) - for s in sources - if isinstance(s, dict) and _safe_float(s.get("value")) is not None - ] - deb_val = _safe_float(row.get("deb_prediction") or (row.get("deb") or {}).get("prediction")) - cities.append( - { - "city": str(city), - "display_name": row.get("display_name") or str(city), - "local_date": row.get("local_date", ""), - "deb": deb_val, - "model_min": min(values) if values else None, - "model_max": max(values) if values else None, - "model_count": len(values), - "current_temp": _safe_float(row.get("current_temp") or (row.get("current") or {}).get("temp")), - "max_so_far": _safe_float(row.get("current_max_so_far") or row.get("max_so_far") or (row.get("current") or {}).get("max_so_far")), - "risk_level": row.get("risk_level", ""), - "temp_unit": row.get("temp_unit") or row.get("temp_symbol") or "°C", - } - ) +def _safe_float(value: Any) -> Optional[float]: + try: + if value is None or value == "": + return None + number = float(value) + except Exception: + return None + return number if number == number else None - system_prompt = ( - "你是 PolyWeather 的天气市场概览员。基于全部城市的扫描数据，写一段今日市场概览。" - "用 3-5 句概括：整体模型一致性、最值得关注的城市（模型分歧大或实测偏离集群）、异常信号。" - "highlights 最多 5 个城市，每个城市一句话点出关键信号。" - "只返回 JSON object，不要 Markdown。所有 *_zh 字段写简体中文，*_en 字段写英文。" - ) - task = ( - "Return JSON: overview_zh, overview_en, highlights (array of {city, note_zh, note_en}, max 5). " - "overview: 3-5 sentences covering model consensus, top divergence cities, anomalies. " - "highlights: per-city one-sentence signal. Keep compact." + +def _row_city(row: Dict[str, Any]) -> str: + return str(row.get("display_name") or row.get("city") or row.get("name") or "").strip() + + +def _row_edge(row: Dict[str, Any]) -> Optional[float]: + return ( + _safe_float(row.get("edge_percent")) + or _safe_float(row.get("edge_pct")) + or _safe_float(row.get("edge")) ) - return { - "model": SCAN_CITY_AI_MODEL, - "temperature": 0.3, - "max_tokens": _OVERVIEW_MAX_TOKENS, - "response_format": {"type": "json_object"}, - "messages": [ - {"role": "system", "content": system_prompt}, - { - "role": "user", - "content": json.dumps( - { - "locale": locale, - "task": task, - "city_count": len(cities), - "cities": cities, - }, - ensure_ascii=False, - default=str, - ), - }, - ], - } + +def _row_score(row: Dict[str, Any]) -> float: + edge = _row_edge(row) or 0.0 + final_score = _safe_float(row.get("final_score")) or 0.0 + liquidity = _safe_float(row.get("liquidity")) or _safe_float(row.get("liquidity_num")) or 0.0 + return edge * 10.0 + final_score + min(liquidity / 1000.0, 25.0) + + +def _row_liquidity(row: Dict[str, Any]) -> float: + return _safe_float(row.get("liquidity")) or _safe_float(row.get("liquidity_num")) or 0.0 + + +def _row_prob_gap(row: Dict[str, Any]) -> Optional[float]: + model_prob = _safe_float(row.get("model_probability")) + market_prob = _safe_float(row.get("market_probability")) + if model_prob is None or market_prob is None: + return None + return model_prob - market_prob def _cache_key(rows: List[Dict[str, Any]], locale: str) -> str: finger = { - "city_ids": sorted( - row.get("city") or row.get("name") or "" + "locale": locale, + "rows": [ + { + "city": row.get("city") or row.get("name") or "", + "edge": _row_edge(row), + "score": _safe_float(row.get("final_score")), + "liquidity": _row_liquidity(row), + "status": row.get("status") or row.get("signal_status") or "", + } for row in rows if isinstance(row, dict) - ), - "locale": locale, + ], } raw = json.dumps(finger, sort_keys=True, ensure_ascii=False, default=str) return "overview:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() +def _build_highlights(rows: List[Dict[str, Any]]) -> List[Dict[str, str]]: + ranked = sorted( + (row for row in rows if isinstance(row, dict) and _row_city(row)), + key=lambda item: (_row_score(item), _row_liquidity(item)), + reverse=True, + ) + highlights: List[Dict[str, str]] = [] + for row in ranked[:5]: + city = _row_city(row) + edge = _row_edge(row) + liquidity = _row_liquidity(row) + gap = _row_prob_gap(row) + edge_text = f"{edge:.1f}%" if edge is not None else "--" + gap_text = f"{gap * 100:.1f}pp" if gap is not None else "--" + liquidity_text = f"{liquidity:,.0f}" if liquidity else "--" + highlights.append( + { + "city": city, + "note_zh": f"edge {edge_text}，模型/市场概率差 {gap_text}，流动性 {liquidity_text}。", + "note_en": f"edge {edge_text}, model-market gap {gap_text}, liquidity {liquidity_text}.", + } + ) + return highlights + + +def _build_payload(rows: List[Dict[str, Any]]) -> Dict[str, Any]: + clean_rows = [row for row in rows if isinstance(row, dict)] + total = len(clean_rows) + tradable = sum(1 for row in clean_rows if not row.get("closed") and not row.get("stale_for_today")) + high_risk = sum( + 1 + for row in clean_rows + if str(row.get("risk_level") or row.get("risk") or "").lower() in {"high", "danger", "red"} + ) + avg_edge_values = [_row_edge(row) for row in clean_rows] + avg_edge_nums = [value for value in avg_edge_values if value is not None] + avg_edge = sum(avg_edge_nums) / len(avg_edge_nums) if avg_edge_nums else 0.0 + total_liquidity = sum(_row_liquidity(row) for row in clean_rows) + highlights = _build_highlights(clean_rows) + + overview_zh = ( + f"当前区域共有 {total} 个天气合约，{tradable} 个可交易；" + f"高风险 {high_risk} 个，平均 edge {avg_edge:.1f}%，总流动性 {total_liquidity:,.0f}。" + "优先查看 edge、final score 与流动性同时靠前的城市。" + ) + overview_en = ( + f"{total} weather contracts are in scope, {tradable} tradable; " + f"{high_risk} high-risk rows, average edge {avg_edge:.1f}%, total liquidity {total_liquidity:,.0f}. " + "Prioritize rows where edge, final score and liquidity align." + ) + + return { + "overview_zh": overview_zh, + "overview_en": overview_en, + "highlights": highlights, + "generated_at": datetime.utcnow().isoformat() + "Z", + "cache_ttl_sec": OVERVIEW_CACHE_TTL_SEC, + "source": "deterministic", + } + + def build_market_overview_payload( rows: List[Dict[str, Any]], *, @@ -116,7 +144,14 @@ def build_market_overview_payload( force_refresh: bool = False, ) -> Dict[str, Any]: if not rows: - return {"overview_zh": "", "overview_en": "", "highlights": [], "generated_at": None} + return { + "overview_zh": "", + "overview_en": "", + "highlights": [], + "generated_at": None, + "cache_ttl_sec": OVERVIEW_CACHE_TTL_SEC, + "source": "deterministic", + } key = _cache_key(rows, locale) if not force_refresh: @@ -125,72 +160,10 @@ def build_market_overview_payload( if cached and cached.get("expires_at", 0) >= time.time(): return cached["payload"] - api_key = _scan_ai_api_key() - if not api_key: - return { - "overview_zh": "AI 概览不可用（未配置 API Key）", - "overview_en": "AI overview unavailable (API key not configured)", - "highlights": [], - "generated_at": datetime.utcnow().isoformat() + "Z", - } - - import httpx - - request_json = _build_overview_ai_request(rows, locale) - generated_at = datetime.utcnow().isoformat() + "Z" - started = time.perf_counter() - - try: - response = httpx.post( - f"{SCAN_AI_BASE_URL}/chat/completions", - json=request_json, - headers={ - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - }, - timeout=min(SCAN_CITY_AI_TIMEOUT_SEC, 15), - ) - response.raise_for_status() - result = response.json() - content = ((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "{}" - parsed = json.loads(content) if isinstance(content, str) else content - if not isinstance(parsed, dict): - raise ValueError("AI returned non-dict overview") - - payload: Dict[str, Any] = { - "overview_zh": str(parsed.get("overview_zh") or parsed.get("overview_en") or ""), - "overview_en": str(parsed.get("overview_en") or parsed.get("overview_zh") or ""), - "highlights": [ - { - "city": str(h.get("city", "")), - "note_zh": str(h.get("note_zh", "")), - "note_en": str(h.get("note_en", "")), - } - for h in (parsed.get("highlights") if isinstance(parsed.get("highlights"), list) else []) - if isinstance(h, dict) - ][:5], - "generated_at": generated_at, - } - except Exception as exc: - logger.warning("Market overview AI failed: {}", exc) - payload = { - "overview_zh": "市场概览暂时无法生成，请稍后刷新。", - "overview_en": "Market overview temporarily unavailable, please refresh later.", - "highlights": [], - "generated_at": generated_at, - } - - duration_ms = int((time.perf_counter() - started) * 1000) - logger.info( - "market_overview cities={} locale={} duration_ms={} cached={}", - len(rows), - locale, - duration_ms, - False, - ) - - entry = {"expires_at": time.time() + _OVERVIEW_CACHE_TTL_SEC, "payload": payload} + payload = _build_payload(rows) with _OVERVIEW_CACHE_LOCK: - _OVERVIEW_CACHE[key] = entry - + _OVERVIEW_CACHE[key] = { + "expires_at": time.time() + OVERVIEW_CACHE_TTL_SEC, + "payload": payload, + } return payload diff --git a/web/services/scan_ai_config.py b/web/services/scan_ai_config.py index a11e97be..e9cfcbab 100644 --- a/web/services/scan_ai_config.py +++ b/web/services/scan_ai_config.py @@ -10,6 +10,8 @@ import os import threading from typing import Any, Dict, Optional +from src.utils.refresh_policy import SCAN_ROWS_REFRESH_SEC + _SCAN_CITY_AI_CACHE_LOCK = threading.Lock() _SCAN_CITY_AI_CACHE: Dict[str, Dict[str, Any]] = {} @@ -32,9 +34,9 @@ def _env_int( return value -SCAN_TERMINAL_PAYLOAD_TTL_SEC = max( - 10, - int(os.getenv("POLYWEATHER_SCAN_TERMINAL_PAYLOAD_TTL_SEC", "120")), +SCAN_TERMINAL_PAYLOAD_TTL_SEC = min( + SCAN_ROWS_REFRESH_SEC, + max(10, int(os.getenv("POLYWEATHER_SCAN_TERMINAL_PAYLOAD_TTL_SEC", str(SCAN_ROWS_REFRESH_SEC)))), ) SCAN_TERMINAL_BUILD_TIMEOUT_SEC = max( 8, diff --git a/web/services/scan_api.py b/web/services/scan_api.py index 5e215790..e9c58622 100644 --- a/web/services/scan_api.py +++ b/web/services/scan_api.py @@ -6,7 +6,6 @@ from typing import Any, Dict from fastapi import HTTPException, Request from fastapi.concurrency import run_in_threadpool -from fastapi.responses import StreamingResponse from web.services.market_overview_api import build_market_overview_payload import web.routes as legacy_routes @@ -84,40 +83,6 @@ async def get_scan_terminal_ai_payload(request: Request) -> Dict[str, Any]: ) -async def get_scan_city_ai_forecast_payload(request: Request) -> Dict[str, Any]: - legacy_routes._assert_entitlement(request) - body = await _json_body_or_empty(request) - city = _extract_required_city(body) - force_refresh = _boolish(body.get("force_refresh")) - locale = str(body.get("locale") or "zh-CN").strip() - return await run_in_threadpool( - legacy_routes.build_scan_city_ai_forecast_payload, - city, - force_refresh=force_refresh, - locale=locale, - ) - - -async def get_scan_city_ai_stream_response(request: Request) -> StreamingResponse: - legacy_routes._assert_entitlement(request) - body = await _json_body_or_empty(request) - city = _extract_required_city(body) - force_refresh = _boolish(body.get("force_refresh")) - locale = str(body.get("locale") or "zh-CN").strip() - return StreamingResponse( - legacy_routes.stream_scan_city_ai_forecast_payload( - city, - force_refresh=force_refresh, - locale=locale, - ), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-store", - "X-Accel-Buffering": "no", - }, - ) - - async def get_scan_terminal_overview_payload(request: Request) -> Dict[str, Any]: try: body = await request.json()