清理市场概览残留：删除 MarketOverviewBanner+refresh_policy 中的 marketOverview 条目

web/services/market_overview_api.py

@@ -1,4 +1,4 @@
-"""Market overview — AI summary of all scan terminal rows, cached 10 min."""
+"""Deterministic market overview for scan terminal rows, cached 10 minutes."""
 
 from __future__ import annotations
 
@@ -7,108 +7,136 @@ import json
 import threading
 import time
 from datetime import datetime
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
-from loguru import logger
-
-from web.scan_city_ai_helpers import _safe_float
-from web.scan_terminal_service import (
-    SCAN_AI_BASE_URL,
-    SCAN_CITY_AI_MODEL,
-    SCAN_CITY_AI_TIMEOUT_SEC,
-    _scan_ai_api_key,
-)
+from src.utils.refresh_policy import MARKET_OVERVIEW_TTL_SEC
 
 _OVERVIEW_CACHE: Dict[str, Dict[str, Any]] = {}
 _OVERVIEW_CACHE_LOCK = threading.Lock()
-_OVERVIEW_MAX_TOKENS = 600
-_OVERVIEW_CACHE_TTL_SEC = 600
+OVERVIEW_CACHE_TTL_SEC = MARKET_OVERVIEW_TTL_SEC
 
 
-def _build_overview_ai_request(
-    rows: List[Dict[str, Any]],
-    locale: str,
-) -> Dict[str, Any]:
-    cities = []
-    for row in rows:
-        if not isinstance(row, dict):
-            continue
-        city = row.get("city") or row.get("name") or ""
-        if not city:
-            continue
-        model_cluster = row.get("model_cluster") if isinstance(row.get("model_cluster"), dict) else {}
-        sources = model_cluster.get("sources") if isinstance(model_cluster.get("sources"), list) else []
-        values = [
-            _safe_float(s.get("value"))
-            for s in sources
-            if isinstance(s, dict) and _safe_float(s.get("value")) is not None
-        ]
-        deb_val = _safe_float(row.get("deb_prediction") or (row.get("deb") or {}).get("prediction"))
-        cities.append(
-            {
-                "city": str(city),
-                "display_name": row.get("display_name") or str(city),
-                "local_date": row.get("local_date", ""),
-                "deb": deb_val,
-                "model_min": min(values) if values else None,
-                "model_max": max(values) if values else None,
-                "model_count": len(values),
-                "current_temp": _safe_float(row.get("current_temp") or (row.get("current") or {}).get("temp")),
-                "max_so_far": _safe_float(row.get("current_max_so_far") or row.get("max_so_far") or (row.get("current") or {}).get("max_so_far")),
-                "risk_level": row.get("risk_level", ""),
-                "temp_unit": row.get("temp_unit") or row.get("temp_symbol") or "°C",
-            }
-        )
+def _safe_float(value: Any) -> Optional[float]:
+    try:
+        if value is None or value == "":
+            return None
+        number = float(value)
+    except Exception:
+        return None
+    return number if number == number else None
 
-    system_prompt = (
-        "你是 PolyWeather 的天气市场概览员。基于全部城市的扫描数据，写一段今日市场概览。"
-        "用 3-5 句概括：整体模型一致性、最值得关注的城市（模型分歧大或实测偏离集群）、异常信号。"
-        "highlights 最多 5 个城市，每个城市一句话点出关键信号。"
-        "只返回 JSON object，不要 Markdown。所有 *_zh 字段写简体中文，*_en 字段写英文。"
-    )
-    task = (
-        "Return JSON: overview_zh, overview_en, highlights (array of {city, note_zh, note_en}, max 5). "
-        "overview: 3-5 sentences covering model consensus, top divergence cities, anomalies. "
-        "highlights: per-city one-sentence signal. Keep compact."
+
+def _row_city(row: Dict[str, Any]) -> str:
+    return str(row.get("display_name") or row.get("city") or row.get("name") or "").strip()
+
+
+def _row_edge(row: Dict[str, Any]) -> Optional[float]:
+    return (
+        _safe_float(row.get("edge_percent"))
+        or _safe_float(row.get("edge_pct"))
+        or _safe_float(row.get("edge"))
     )
 
-    return {
-        "model": SCAN_CITY_AI_MODEL,
-        "temperature": 0.3,
-        "max_tokens": _OVERVIEW_MAX_TOKENS,
-        "response_format": {"type": "json_object"},
-        "messages": [
-            {"role": "system", "content": system_prompt},
-            {
-                "role": "user",
-                "content": json.dumps(
-                    {
-                        "locale": locale,
-                        "task": task,
-                        "city_count": len(cities),
-                        "cities": cities,
-                    },
-                    ensure_ascii=False,
-                    default=str,
-                ),
-            },
-        ],
-    }
+
+def _row_score(row: Dict[str, Any]) -> float:
+    edge = _row_edge(row) or 0.0
+    final_score = _safe_float(row.get("final_score")) or 0.0
+    liquidity = _safe_float(row.get("liquidity")) or _safe_float(row.get("liquidity_num")) or 0.0
+    return edge * 10.0 + final_score + min(liquidity / 1000.0, 25.0)
+
+
+def _row_liquidity(row: Dict[str, Any]) -> float:
+    return _safe_float(row.get("liquidity")) or _safe_float(row.get("liquidity_num")) or 0.0
+
+
+def _row_prob_gap(row: Dict[str, Any]) -> Optional[float]:
+    model_prob = _safe_float(row.get("model_probability"))
+    market_prob = _safe_float(row.get("market_probability"))
+    if model_prob is None or market_prob is None:
+        return None
+    return model_prob - market_prob
 
 
 def _cache_key(rows: List[Dict[str, Any]], locale: str) -> str:
     finger = {
-        "city_ids": sorted(
-            row.get("city") or row.get("name") or ""
+        "locale": locale,
+        "rows": [
+            {
+                "city": row.get("city") or row.get("name") or "",
+                "edge": _row_edge(row),
+                "score": _safe_float(row.get("final_score")),
+                "liquidity": _row_liquidity(row),
+                "status": row.get("status") or row.get("signal_status") or "",
+            }
             for row in rows
             if isinstance(row, dict)
-        ),
-        "locale": locale,
+        ],
     }
     raw = json.dumps(finger, sort_keys=True, ensure_ascii=False, default=str)
     return "overview:" + hashlib.sha256(raw.encode("utf-8")).hexdigest()
 
 
+def _build_highlights(rows: List[Dict[str, Any]]) -> List[Dict[str, str]]:
+    ranked = sorted(
+        (row for row in rows if isinstance(row, dict) and _row_city(row)),
+        key=lambda item: (_row_score(item), _row_liquidity(item)),
+        reverse=True,
+    )
+    highlights: List[Dict[str, str]] = []
+    for row in ranked[:5]:
+        city = _row_city(row)
+        edge = _row_edge(row)
+        liquidity = _row_liquidity(row)
+        gap = _row_prob_gap(row)
+        edge_text = f"{edge:.1f}%" if edge is not None else "--"
+        gap_text = f"{gap * 100:.1f}pp" if gap is not None else "--"
+        liquidity_text = f"{liquidity:,.0f}" if liquidity else "--"
+        highlights.append(
+            {
+                "city": city,
+                "note_zh": f"edge {edge_text}，模型/市场概率差 {gap_text}，流动性 {liquidity_text}。",
+                "note_en": f"edge {edge_text}, model-market gap {gap_text}, liquidity {liquidity_text}.",
+            }
+        )
+    return highlights
+
+
+def _build_payload(rows: List[Dict[str, Any]]) -> Dict[str, Any]:
+    clean_rows = [row for row in rows if isinstance(row, dict)]
+    total = len(clean_rows)
+    tradable = sum(1 for row in clean_rows if not row.get("closed") and not row.get("stale_for_today"))
+    high_risk = sum(
+        1
+        for row in clean_rows
+        if str(row.get("risk_level") or row.get("risk") or "").lower() in {"high", "danger", "red"}
+    )
+    avg_edge_values = [_row_edge(row) for row in clean_rows]
+    avg_edge_nums = [value for value in avg_edge_values if value is not None]
+    avg_edge = sum(avg_edge_nums) / len(avg_edge_nums) if avg_edge_nums else 0.0
+    total_liquidity = sum(_row_liquidity(row) for row in clean_rows)
+    highlights = _build_highlights(clean_rows)
+
+    overview_zh = (
+        f"当前区域共有 {total} 个天气合约，{tradable} 个可交易；"
+        f"高风险 {high_risk} 个，平均 edge {avg_edge:.1f}%，总流动性 {total_liquidity:,.0f}。"
+        "优先查看 edge、final score 与流动性同时靠前的城市。"
+    )
+    overview_en = (
+        f"{total} weather contracts are in scope, {tradable} tradable; "
+        f"{high_risk} high-risk rows, average edge {avg_edge:.1f}%, total liquidity {total_liquidity:,.0f}. "
+        "Prioritize rows where edge, final score and liquidity align."
+    )
+
+    return {
+        "overview_zh": overview_zh,
+        "overview_en": overview_en,
+        "highlights": highlights,
+        "generated_at": datetime.utcnow().isoformat() + "Z",
+        "cache_ttl_sec": OVERVIEW_CACHE_TTL_SEC,
+        "source": "deterministic",
+    }
+
+
 def build_market_overview_payload(
     rows: List[Dict[str, Any]],
     *,
@@ -116,7 +144,14 @@ def build_market_overview_payload(
     force_refresh: bool = False,
 ) -> Dict[str, Any]:
     if not rows:
-        return {"overview_zh": "", "overview_en": "", "highlights": [], "generated_at": None}
+        return {
+            "overview_zh": "",
+            "overview_en": "",
+            "highlights": [],
+            "generated_at": None,
+            "cache_ttl_sec": OVERVIEW_CACHE_TTL_SEC,
+            "source": "deterministic",
+        }
 
     key = _cache_key(rows, locale)
     if not force_refresh:
@@ -125,72 +160,10 @@ def build_market_overview_payload(
             if cached and cached.get("expires_at", 0) >= time.time():
                 return cached["payload"]
 
-    api_key = _scan_ai_api_key()
-    if not api_key:
-        return {
-            "overview_zh": "AI 概览不可用（未配置 API Key）",
-            "overview_en": "AI overview unavailable (API key not configured)",
-            "highlights": [],
-            "generated_at": datetime.utcnow().isoformat() + "Z",
-        }
-
-    import httpx
-
-    request_json = _build_overview_ai_request(rows, locale)
-    generated_at = datetime.utcnow().isoformat() + "Z"
-    started = time.perf_counter()
-
-    try:
-        response = httpx.post(
-            f"{SCAN_AI_BASE_URL}/chat/completions",
-            json=request_json,
-            headers={
-                "Authorization": f"Bearer {api_key}",
-                "Content-Type": "application/json",
-            },
-            timeout=min(SCAN_CITY_AI_TIMEOUT_SEC, 15),
-        )
-        response.raise_for_status()
-        result = response.json()
-        content = ((result.get("choices") or [{}])[0].get("message") or {}).get("content") or "{}"
-        parsed = json.loads(content) if isinstance(content, str) else content
-        if not isinstance(parsed, dict):
-            raise ValueError("AI returned non-dict overview")
-
-        payload: Dict[str, Any] = {
-            "overview_zh": str(parsed.get("overview_zh") or parsed.get("overview_en") or ""),
-            "overview_en": str(parsed.get("overview_en") or parsed.get("overview_zh") or ""),
-            "highlights": [
-                {
-                    "city": str(h.get("city", "")),
-                    "note_zh": str(h.get("note_zh", "")),
-                    "note_en": str(h.get("note_en", "")),
-                }
-                for h in (parsed.get("highlights") if isinstance(parsed.get("highlights"), list) else [])
-                if isinstance(h, dict)
-            ][:5],
-            "generated_at": generated_at,
-        }
-    except Exception as exc:
-        logger.warning("Market overview AI failed: {}", exc)
-        payload = {
-            "overview_zh": "市场概览暂时无法生成，请稍后刷新。",
-            "overview_en": "Market overview temporarily unavailable, please refresh later.",
-            "highlights": [],
-            "generated_at": generated_at,
-        }
-
-    duration_ms = int((time.perf_counter() - started) * 1000)
-    logger.info(
-        "market_overview cities={} locale={} duration_ms={} cached={}",
-        len(rows),
-        locale,
-        duration_ms,
-        False,
-    )
-
-    entry = {"expires_at": time.time() + _OVERVIEW_CACHE_TTL_SEC, "payload": payload}
+    payload = _build_payload(rows)
     with _OVERVIEW_CACHE_LOCK:
-        _OVERVIEW_CACHE[key] = entry
-
+        _OVERVIEW_CACHE[key] = {
+            "expires_at": time.time() + OVERVIEW_CACHE_TTL_SEC,
+            "payload": payload,
+        }
     return payload