sync Cloudflare rules and cache scan snapshots

.github/workflows/ci.yml

@@ -138,3 +138,13 @@ jobs:
           printf '%s\n' "$GHCR_PAT" | ssh -o StrictHostKeyChecking=accept-new ${{ secrets.VPS_USER }}@${{ secrets.VPS_HOST }} "
             bash /tmp/deploy.sh '${{ github.sha }}'
           "
+
+      - name: Apply Cloudflare cache rules
+        env:
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+        run: |
+          if [ -z "${CLOUDFLARE_API_TOKEN}" ]; then
+            echo "CLOUDFLARE_API_TOKEN is not configured; skipping Cache Rules sync"
+            exit 0
+          fi
+          python scripts/configure_cloudflare_free.py --apply

docs/CLOUDFLARE_FREE_CACHE_ZH.md

@@ -20,9 +20,48 @@
 
 ## Cache Rules
 
-按以下顺序创建，绕过规则必须放在公开缓存规则之前。免费版规则数量有限，因此使用路径集合合并表达式。
+按以下顺序创建。Cloudflare 同一阶段最后匹配的规则生效，因此绕过规则必须放在公开缓存规则之后。免费版规则数量有限，因此使用路径集合合并表达式。
 
-### 1. 绕过后端域名、动态和敏感请求
+也可以使用仓库内脚本自动创建或更新规则。脚本会保留非 PolyWeather 规则，并把绕过规则放在最后：
+
+```powershell
+$env:CLOUDFLARE_API_TOKEN="<具有 Cache Rules Edit 权限的 token>"
+python scripts/configure_cloudflare_free.py
+python scripts/configure_cloudflare_free.py --apply
+```
+
+第一条命令只输出计划；只有带 `--apply` 才会修改 Cloudflare。
+
+部署流水线也会执行同一脚本。给 GitHub 仓库增加名为 `CLOUDFLARE_API_TOKEN` 的 Secret 后，后续每次成功部署都会同步 Cache Rules；未配置时流水线会明确跳过。
+
+### 1. 缓存公开内容
+
+动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。
+
+把下面的公开页面、静态资源和公开数据接口合并成一条规则即可：
+
+```text
+http.host eq "polyweather.top"
+and http.request.method in {"GET" "HEAD"}
+and (
+  http.request.uri.path eq "/"
+  or starts_with(http.request.uri.path, "/_next/static/")
+  or starts_with(http.request.uri.path, "/docs/")
+  or starts_with(http.request.uri.path, "/modern/")
+  or starts_with(http.request.uri.path, "/probabilities/")
+  or starts_with(http.request.uri.path, "/subscription-help/")
+  or http.request.uri.path eq "/api/cities"
+  or http.request.uri.path eq "/api/cities/detail-batch"
+  or starts_with(http.request.uri.path, "/api/city/")
+  or http.request.uri.path eq "/api/scan/terminal"
+  or http.request.uri.path eq "/api/system/status"
+  or lower(http.request.uri.path.extension) in {
+    "js" "css" "woff" "woff2" "png" "jpg" "jpeg" "webp" "avif" "svg" "ico"
+  }
+)
+```
+
+### 2. 最后绕过后端域名、动态和敏感请求
 
 动作：Bypass cache
 
@@ -51,7 +90,9 @@ or any(http.request.headers["authorization"][*] ne "")
 or http.cookie contains "sb-"
 ```
 
-### 2. 缓存静态资源
+## 缓存 TTL
+
+### 静态资源
 
 动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。
 
@@ -69,7 +110,7 @@ and (
 
 源站 TTL：一年 immutable。
 
-### 3. 缓存公开页面
+### 公开页面
 
 动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。
 
@@ -88,7 +129,7 @@ and (
 
 源站 TTL：10 分钟，过期后允许后台刷新 1 小时。
 
-### 4. 缓存公开数据接口
+### 公开数据接口
 
 动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。
 

scripts/__init__.py

@@ -0,0 +1 @@
+"""Operational scripts that are also covered by focused unit tests."""

scripts/configure_cloudflare_free.py

@@ -0,0 +1,218 @@
+"""Configure PolyWeather's Cloudflare Free cache rules.
+
+The script is intentionally conservative:
+- It preserves rules that it does not own.
+- It keeps the sensitive-request bypass rule last because the last matching
+  Cloudflare Cache Rule wins.
+- It only mutates Cloudflare when --apply is provided.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from typing import Any, Dict, Iterable, List
+from urllib.error import HTTPError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+API_BASE = "https://api.cloudflare.com/client/v4"
+MANAGED_RULE_REF_PREFIX = "polyweather_free_cache_"
+PHASE = "http_request_cache_settings"
+
+PUBLIC_CACHE_EXPRESSION = """(
+  http.host eq "polyweather.top"
+  and http.request.method in {"GET" "HEAD"}
+  and (
+    http.request.uri.path eq "/"
+    or starts_with(http.request.uri.path, "/_next/static/")
+    or starts_with(http.request.uri.path, "/docs/")
+    or starts_with(http.request.uri.path, "/modern/")
+    or starts_with(http.request.uri.path, "/probabilities/")
+    or starts_with(http.request.uri.path, "/subscription-help/")
+    or http.request.uri.path eq "/api/cities"
+    or http.request.uri.path eq "/api/cities/detail-batch"
+    or starts_with(http.request.uri.path, "/api/city/")
+    or http.request.uri.path eq "/api/scan/terminal"
+    or http.request.uri.path eq "/api/system/status"
+    or lower(http.request.uri.path.extension) in {
+      "js" "css" "woff" "woff2" "png" "jpg" "jpeg" "webp" "avif" "svg" "ico"
+    }
+  )
+)"""
+
+BYPASS_CACHE_EXPRESSION = """(
+  http.host eq "api.polyweather.top"
+  or (
+    http.host eq "polyweather.top"
+    and (
+      (http.request.method ne "GET" and http.request.method ne "HEAD")
+      or starts_with(http.request.uri.path, "/api/auth/")
+      or starts_with(http.request.uri.path, "/api/feedback")
+      or starts_with(http.request.uri.path, "/api/events")
+      or starts_with(http.request.uri.path, "/api/internal/")
+      or starts_with(http.request.uri.path, "/api/ops/")
+      or starts_with(http.request.uri.path, "/api/payments/")
+      or starts_with(http.request.uri.path, "/account")
+      or starts_with(http.request.uri.path, "/auth")
+      or starts_with(http.request.uri.path, "/ops")
+      or starts_with(http.request.uri.path, "/terminal")
+      or http.request.uri.query contains "force_refresh=true"
+    )
+  )
+)"""
+
+_RULE_FIELDS = {
+    "action",
+    "action_parameters",
+    "description",
+    "enabled",
+    "expression",
+    "logging",
+    "ratelimit",
+    "ref",
+}
+
+
+def _compact_expression(expression: str) -> str:
+    return " ".join(line.strip() for line in expression.splitlines() if line.strip())
+
+
+def build_managed_cache_rules() -> List[Dict[str, Any]]:
+    return [
+        {
+            "ref": f"{MANAGED_RULE_REF_PREFIX}public",
+            "description": "PolyWeather: cache public pages, assets, and public data APIs",
+            "expression": _compact_expression(PUBLIC_CACHE_EXPRESSION),
+            "action": "set_cache_settings",
+            "action_parameters": {"cache": True},
+            "enabled": True,
+        },
+        {
+            "ref": f"{MANAGED_RULE_REF_PREFIX}bypass",
+            "description": "PolyWeather: bypass backend, sensitive, realtime, and force-refresh requests",
+            "expression": _compact_expression(BYPASS_CACHE_EXPRESSION),
+            "action": "set_cache_settings",
+            "action_parameters": {"cache": False},
+            "enabled": True,
+        },
+    ]
+
+
+def _portable_rule(rule: Dict[str, Any]) -> Dict[str, Any]:
+    return {key: rule[key] for key in _RULE_FIELDS if key in rule}
+
+
+def merge_managed_rules(
+    existing_rules: Iterable[Dict[str, Any]],
+    managed_rules: Iterable[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+    unmanaged = [
+        _portable_rule(rule)
+        for rule in existing_rules
+        if not str(rule.get("ref") or "").startswith(MANAGED_RULE_REF_PREFIX)
+    ]
+    return [*unmanaged, *[_portable_rule(rule) for rule in managed_rules]]
+
+
+class CloudflareApi:
+    def __init__(self, token: str):
+        self.token = token
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        payload: Dict[str, Any] | None = None,
+        *,
+        allow_not_found: bool = False,
+    ) -> Dict[str, Any] | None:
+        body = None if payload is None else json.dumps(payload).encode("utf-8")
+        request = Request(
+            f"{API_BASE}{path}",
+            data=body,
+            method=method,
+            headers={
+                "Authorization": f"Bearer {self.token}",
+                "Content-Type": "application/json",
+                "User-Agent": "polyweather-cloudflare-config/1.0",
+            },
+        )
+        try:
+            with urlopen(request, timeout=30) as response:
+                result = json.loads(response.read().decode("utf-8"))
+        except HTTPError as exc:
+            if allow_not_found and exc.code == 404:
+                return None
+            detail = exc.read().decode("utf-8", errors="replace")
+            raise RuntimeError(f"Cloudflare API {method} {path} failed: {exc.code} {detail}") from exc
+        if not result.get("success"):
+            raise RuntimeError(f"Cloudflare API {method} {path} failed: {result.get('errors')}")
+        return result
+
+
+def resolve_zone_id(api: CloudflareApi, zone_name: str, explicit_zone_id: str) -> str:
+    if explicit_zone_id:
+        return explicit_zone_id
+    result = api.request("GET", f"/zones?{urlencode({'name': zone_name, 'status': 'active'})}")
+    zones = list((result or {}).get("result") or [])
+    if len(zones) != 1:
+        raise RuntimeError(f"Expected one active Cloudflare zone named {zone_name}, found {len(zones)}")
+    return str(zones[0]["id"])
+
+
+def apply_cache_rules(api: CloudflareApi, zone_id: str) -> List[Dict[str, Any]]:
+    path = f"/zones/{zone_id}/rulesets/phases/{PHASE}/entrypoint"
+    current = api.request("GET", path, allow_not_found=True) or {}
+    existing_rules = list((current.get("result") or {}).get("rules") or [])
+    merged_rules = merge_managed_rules(existing_rules, build_managed_cache_rules())
+    api.request(
+        "PUT",
+        path,
+        {
+            "description": "PolyWeather zone-level cache rules",
+            "rules": merged_rules,
+        },
+    )
+    return merged_rules
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apply", action="store_true", help="Apply rules to Cloudflare")
+    parser.add_argument("--zone-name", default="polyweather.top")
+    parser.add_argument("--zone-id", default=os.getenv("CLOUDFLARE_ZONE_ID", ""))
+    parser.add_argument("--api-token", default=os.getenv("CLOUDFLARE_API_TOKEN", ""))
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = _parse_args()
+    managed_rules = build_managed_cache_rules()
+    if not args.apply:
+        print(json.dumps({"mode": "plan", "managed_rules": managed_rules}, indent=2))
+        return 0
+    if not args.api_token:
+        print("CLOUDFLARE_API_TOKEN or --api-token is required with --apply", file=sys.stderr)
+        return 2
+    api = CloudflareApi(args.api_token)
+    zone_id = resolve_zone_id(api, args.zone_name, args.zone_id)
+    merged_rules = apply_cache_rules(api, zone_id)
+    print(
+        json.dumps(
+            {
+                "mode": "applied",
+                "zone_id": zone_id,
+                "managed_rule_count": len(managed_rules),
+                "total_rule_count": len(merged_rules),
+            },
+            indent=2,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_cloudflare_free_config.py

@@ -0,0 +1,48 @@
+from scripts.configure_cloudflare_free import (
+    MANAGED_RULE_REF_PREFIX,
+    build_managed_cache_rules,
+    merge_managed_rules,
+)
+
+
+def test_cloudflare_managed_rules_cache_public_content_then_bypass_sensitive_requests():
+    rules = build_managed_cache_rules()
+
+    assert [rule["ref"] for rule in rules] == [
+        f"{MANAGED_RULE_REF_PREFIX}public",
+        f"{MANAGED_RULE_REF_PREFIX}bypass",
+    ]
+    assert rules[0]["action_parameters"]["cache"] is True
+    assert rules[-1]["action_parameters"]["cache"] is False
+    assert 'http.host eq "api.polyweather.top"' in rules[-1]["expression"]
+    assert 'http.request.uri.query contains "force_refresh=true"' in rules[-1]["expression"]
+
+
+def test_cloudflare_rule_merge_preserves_unmanaged_rules_and_puts_bypass_last():
+    existing = [
+        {
+            "ref": "existing_rule",
+            "description": "keep me",
+            "expression": 'http.host eq "example.com"',
+            "action": "set_cache_settings",
+            "action_parameters": {"cache": True},
+            "enabled": True,
+        },
+        {
+            "ref": f"{MANAGED_RULE_REF_PREFIX}old",
+            "description": "replace me",
+            "expression": "true",
+            "action": "set_cache_settings",
+            "action_parameters": {"cache": False},
+            "enabled": True,
+        },
+    ]
+
+    merged = merge_managed_rules(existing, build_managed_cache_rules())
+
+    assert [rule["ref"] for rule in merged] == [
+        "existing_rule",
+        f"{MANAGED_RULE_REF_PREFIX}public",
+        f"{MANAGED_RULE_REF_PREFIX}bypass",
+    ]
+    assert merged[-1]["action_parameters"]["cache"] is False

tests/test_deployment_runtime_config.py

@@ -138,6 +138,13 @@ def test_scan_terminal_backend_timeout_returns_before_next_proxy_abort():
     )
 
 
+def test_deploy_workflow_applies_cloudflare_rules_when_token_is_available():
+    workflow = (ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8")
+
+    assert "CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}" in workflow
+    assert "python scripts/configure_cloudflare_free.py --apply" in workflow
+
+
 def test_probability_engine_uses_enriched_multi_model_snapshot():
     source = (ROOT / "web" / "analysis_service.py").read_text(encoding="utf-8")
 

tests/test_scan_terminal_modules.py

@@ -10,6 +10,7 @@ from web.scan_terminal_payloads import (
     SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS,
 )
 from web.scan_terminal_ranker import build_ranked_scan_terminal_result
+from web import scan_terminal_city_row
 from web.scan_terminal_city_row import _build_quick_row
 from web.routers.scan import router as scan_router
 from web.scan_terminal_service import _scan_terminal_prewarm_filters
@@ -81,6 +82,43 @@ def test_scan_terminal_prewarm_covers_default_api_limit():
     assert 180 in limits
 
 
+def test_scan_city_terminal_rows_reuses_persisted_panel_cache(monkeypatch):
+    payload = {
+        "display_name": "Paris",
+        "local_date": "2026-06-10",
+        "local_time": "12:00",
+        "current": {"temp": 18.0},
+        "risk": {},
+        "deb": {"prediction": 20.0},
+        "probabilities": {},
+        "multi_model": {},
+    }
+
+    class _Cache:
+        @staticmethod
+        def get_city_cache(kind, city):
+            assert (kind, city) == ("panel", "paris")
+            return {"payload": payload}
+
+    monkeypatch.setattr(scan_terminal_city_row, "_PANEL_CACHE_DB", _Cache())
+    monkeypatch.setattr(
+        scan_terminal_city_row,
+        "_analyze",
+        lambda *_args, **_kwargs: (_ for _ in ()).throw(
+            AssertionError("non-force scan must not fetch external sources")
+        ),
+    )
+
+    result = scan_terminal_city_row._scan_city_terminal_rows(
+        "paris",
+        {"market_type": "maxtemp"},
+        force_refresh=False,
+    )
+
+    assert result["city"] == "paris"
+    assert result["rows"][0]["current_temp"] == 18.0
+
+
 def test_scan_router_does_not_expose_terminal_ai_endpoint():
     routes = {
         getattr(route, "path", None): getattr(route, "methods", set())

tests/test_web_observability.py

@@ -3398,6 +3398,41 @@ def test_scan_terminal_cold_requests_start_background_build_without_blocking(mon
     assert all("初始化" in result["stale_reason"] or "刷新中" in result["stale_reason"] for result in results)
 
 
+def test_scan_terminal_background_refresh_reuses_cached_city_data(monkeypatch):
+    filters = {"scan_mode": "tradable", "limit": 17, "min_edge_pct": 6.75}
+    calls = []
+
+    monkeypatch.setattr(
+        scan_terminal_service,
+        "mark_scan_terminal_refreshing",
+        lambda _filters: True,
+    )
+    monkeypatch.setattr(
+        scan_terminal_service,
+        "clear_scan_terminal_refreshing",
+        lambda _filters: None,
+    )
+    monkeypatch.setattr(
+        scan_terminal_service,
+        "_build_scan_terminal_payload_singleflight",
+        lambda filters_arg, *, force_refresh=False: calls.append(
+            (dict(filters_arg), force_refresh)
+        ),
+    )
+
+    class _ImmediateThread:
+        def __init__(self, *, target, name, daemon):
+            self._target = target
+
+        def start(self):
+            self._target()
+
+    monkeypatch.setattr(scan_terminal_service.threading, "Thread", _ImmediateThread)
+
+    assert scan_terminal_service._start_scan_terminal_background_refresh(filters) is True
+    assert calls == [(filters, False)]
+
+
 def test_scan_terminal_nonforce_ignores_ancient_success_snapshot(monkeypatch):
     filters = {"scan_mode": "tradable", "limit": 17, "min_edge_pct": 6.75}
     old_success_t = 1780839484.0

web/scan_terminal_city_row.py

@@ -5,6 +5,7 @@ import re
 from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional
 
+from src.database.db_manager import DBManager
 from web.core import CITIES, _sf as _safe_float
 from web.analysis_service import _analyze
 from web.scan_terminal_filters import (
@@ -16,6 +17,7 @@ from web.services.city_payloads import aggregate_runway_history
 
 SCAN_ROW_RUNWAY_HISTORY_RESOLUTION = "10m"
 SCAN_ROW_MAX_RUNWAY_POINTS = 144
+_PANEL_CACHE_DB = DBManager()
 
 
 def _compact_runway_plate_history_for_scan(raw_history: Any) -> Dict[str, List[Dict[str, Any]]]:
@@ -171,11 +173,18 @@ def _scan_city_terminal_rows_quick(
 ) -> Dict[str, Any]:
     """Fast path that returns cached analysis rows only — returns a single row per city
     with cached analysis data (Obs, DEB, probabilities) but no market prices."""
-    data = _analyze(
+    data: Dict[str, Any] = {}
-        city,
+    if not force_refresh:
-        force_refresh=force_refresh,
+        cached_entry = _PANEL_CACHE_DB.get_city_cache("panel", city)
-        detail_mode="panel",
+        cached_payload = cached_entry.get("payload") if isinstance(cached_entry, dict) else None
-    )
+        if isinstance(cached_payload, dict):
+            data = cached_payload
+    if not data:
+        data = _analyze(
+            city,
+            force_refresh=force_refresh,
+            detail_mode="panel",
+        )
     row = _build_quick_row(city=city, data=data)
     return {
         "city": city,

web/scan_terminal_service.py

@@ -104,7 +104,7 @@ def _start_scan_terminal_background_refresh(filters: Dict[str, Any]) -> bool:
 
     def _runner() -> None:
         try:
-            _build_scan_terminal_payload_singleflight(filters, force_refresh=True)
+            _build_scan_terminal_payload_singleflight(filters, force_refresh=False)
         except Exception as exc:  # pragma: no cover - defensive background guard
             logger.warning("scan terminal background refresh failed: {}", exc)
         finally: