From 25bd077b25f2b5130def6742f33a75ed3d62a66e Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Wed, 10 Jun 2026 19:40:54 +0800 Subject: [PATCH] sync Cloudflare rules and cache scan snapshots --- .github/workflows/ci.yml | 10 ++ docs/CLOUDFLARE_FREE_CACHE_ZH.md | 51 +++++- scripts/__init__.py | 1 + scripts/configure_cloudflare_free.py | 218 ++++++++++++++++++++++++ tests/test_cloudflare_free_config.py | 48 ++++++ tests/test_deployment_runtime_config.py | 7 + tests/test_scan_terminal_modules.py | 38 +++++ tests/test_web_observability.py | 35 ++++ web/scan_terminal_city_row.py | 19 ++- web/scan_terminal_service.py | 2 +- 10 files changed, 418 insertions(+), 11 deletions(-) create mode 100644 scripts/__init__.py create mode 100644 scripts/configure_cloudflare_free.py create mode 100644 tests/test_cloudflare_free_config.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index feee600b..fe8df305 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,3 +138,13 @@ jobs: printf '%s\n' "$GHCR_PAT" | ssh -o StrictHostKeyChecking=accept-new ${{ secrets.VPS_USER }}@${{ secrets.VPS_HOST }} " bash /tmp/deploy.sh '${{ github.sha }}' " + + - name: Apply Cloudflare cache rules + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + run: | + if [ -z "${CLOUDFLARE_API_TOKEN}" ]; then + echo "CLOUDFLARE_API_TOKEN is not configured; skipping Cache Rules sync" + exit 0 + fi + python scripts/configure_cloudflare_free.py --apply diff --git a/docs/CLOUDFLARE_FREE_CACHE_ZH.md b/docs/CLOUDFLARE_FREE_CACHE_ZH.md index f733865c..637cdae3 100644 --- a/docs/CLOUDFLARE_FREE_CACHE_ZH.md +++ b/docs/CLOUDFLARE_FREE_CACHE_ZH.md @@ -20,9 +20,48 @@ ## Cache Rules -按以下顺序创建，绕过规则必须放在公开缓存规则之前。免费版规则数量有限，因此使用路径集合合并表达式。 +按以下顺序创建。Cloudflare 同一阶段最后匹配的规则生效，因此绕过规则必须放在公开缓存规则之后。免费版规则数量有限，因此使用路径集合合并表达式。 -### 1. 绕过后端域名、动态和敏感请求 +也可以使用仓库内脚本自动创建或更新规则。脚本会保留非 PolyWeather 规则，并把绕过规则放在最后： + +```powershell +$env:CLOUDFLARE_API_TOKEN="<具有 Cache Rules Edit 权限的 token>" +python scripts/configure_cloudflare_free.py +python scripts/configure_cloudflare_free.py --apply +``` + +第一条命令只输出计划；只有带 `--apply` 才会修改 Cloudflare。 + +部署流水线也会执行同一脚本。给 GitHub 仓库增加名为 `CLOUDFLARE_API_TOKEN` 的 Secret 后，后续每次成功部署都会同步 Cache Rules；未配置时流水线会明确跳过。 + +### 1. 缓存公开内容 + +动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。 + +把下面的公开页面、静态资源和公开数据接口合并成一条规则即可： + +```text +http.host eq "polyweather.top" +and http.request.method in {"GET" "HEAD"} +and ( + http.request.uri.path eq "/" + or starts_with(http.request.uri.path, "/_next/static/") + or starts_with(http.request.uri.path, "/docs/") + or starts_with(http.request.uri.path, "/modern/") + or starts_with(http.request.uri.path, "/probabilities/") + or starts_with(http.request.uri.path, "/subscription-help/") + or http.request.uri.path eq "/api/cities" + or http.request.uri.path eq "/api/cities/detail-batch" + or starts_with(http.request.uri.path, "/api/city/") + or http.request.uri.path eq "/api/scan/terminal" + or http.request.uri.path eq "/api/system/status" + or lower(http.request.uri.path.extension) in { + "js" "css" "woff" "woff2" "png" "jpg" "jpeg" "webp" "avif" "svg" "ico" + } +) +``` + +### 2. 最后绕过后端域名、动态和敏感请求 动作：Bypass cache @@ -51,7 +90,9 @@ or any(http.request.headers["authorization"][*] ne "") or http.cookie contains "sb-" ``` -### 2. 缓存静态资源 +## 缓存 TTL + +### 静态资源 动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。 @@ -69,7 +110,7 @@ and ( 源站 TTL：一年 immutable。 -### 3. 缓存公开页面 +### 公开页面 动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。 @@ -88,7 +129,7 @@ and ( 源站 TTL：10 分钟，过期后允许后台刷新 1 小时。 -### 4. 缓存公开数据接口 +### 公开数据接口 动作：Eligible for cache；Edge TTL 使用源站 Cache-Control。 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..d087bba1 --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1 @@ +"""Operational scripts that are also covered by focused unit tests.""" diff --git a/scripts/configure_cloudflare_free.py b/scripts/configure_cloudflare_free.py new file mode 100644 index 00000000..d675bf09 --- /dev/null +++ b/scripts/configure_cloudflare_free.py @@ -0,0 +1,218 @@ +"""Configure PolyWeather's Cloudflare Free cache rules. + +The script is intentionally conservative: +- It preserves rules that it does not own. +- It keeps the sensitive-request bypass rule last because the last matching + Cloudflare Cache Rule wins. +- It only mutates Cloudflare when --apply is provided. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from typing import Any, Dict, Iterable, List +from urllib.error import HTTPError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +API_BASE = "https://api.cloudflare.com/client/v4" +MANAGED_RULE_REF_PREFIX = "polyweather_free_cache_" +PHASE = "http_request_cache_settings" + +PUBLIC_CACHE_EXPRESSION = """( + http.host eq "polyweather.top" + and http.request.method in {"GET" "HEAD"} + and ( + http.request.uri.path eq "/" + or starts_with(http.request.uri.path, "/_next/static/") + or starts_with(http.request.uri.path, "/docs/") + or starts_with(http.request.uri.path, "/modern/") + or starts_with(http.request.uri.path, "/probabilities/") + or starts_with(http.request.uri.path, "/subscription-help/") + or http.request.uri.path eq "/api/cities" + or http.request.uri.path eq "/api/cities/detail-batch" + or starts_with(http.request.uri.path, "/api/city/") + or http.request.uri.path eq "/api/scan/terminal" + or http.request.uri.path eq "/api/system/status" + or lower(http.request.uri.path.extension) in { + "js" "css" "woff" "woff2" "png" "jpg" "jpeg" "webp" "avif" "svg" "ico" + } + ) +)""" + +BYPASS_CACHE_EXPRESSION = """( + http.host eq "api.polyweather.top" + or ( + http.host eq "polyweather.top" + and ( + (http.request.method ne "GET" and http.request.method ne "HEAD") + or starts_with(http.request.uri.path, "/api/auth/") + or starts_with(http.request.uri.path, "/api/feedback") + or starts_with(http.request.uri.path, "/api/events") + or starts_with(http.request.uri.path, "/api/internal/") + or starts_with(http.request.uri.path, "/api/ops/") + or starts_with(http.request.uri.path, "/api/payments/") + or starts_with(http.request.uri.path, "/account") + or starts_with(http.request.uri.path, "/auth") + or starts_with(http.request.uri.path, "/ops") + or starts_with(http.request.uri.path, "/terminal") + or http.request.uri.query contains "force_refresh=true" + ) + ) +)""" + +_RULE_FIELDS = { + "action", + "action_parameters", + "description", + "enabled", + "expression", + "logging", + "ratelimit", + "ref", +} + + +def _compact_expression(expression: str) -> str: + return " ".join(line.strip() for line in expression.splitlines() if line.strip()) + + +def build_managed_cache_rules() -> List[Dict[str, Any]]: + return [ + { + "ref": f"{MANAGED_RULE_REF_PREFIX}public", + "description": "PolyWeather: cache public pages, assets, and public data APIs", + "expression": _compact_expression(PUBLIC_CACHE_EXPRESSION), + "action": "set_cache_settings", + "action_parameters": {"cache": True}, + "enabled": True, + }, + { + "ref": f"{MANAGED_RULE_REF_PREFIX}bypass", + "description": "PolyWeather: bypass backend, sensitive, realtime, and force-refresh requests", + "expression": _compact_expression(BYPASS_CACHE_EXPRESSION), + "action": "set_cache_settings", + "action_parameters": {"cache": False}, + "enabled": True, + }, + ] + + +def _portable_rule(rule: Dict[str, Any]) -> Dict[str, Any]: + return {key: rule[key] for key in _RULE_FIELDS if key in rule} + + +def merge_managed_rules( + existing_rules: Iterable[Dict[str, Any]], + managed_rules: Iterable[Dict[str, Any]], +) -> List[Dict[str, Any]]: + unmanaged = [ + _portable_rule(rule) + for rule in existing_rules + if not str(rule.get("ref") or "").startswith(MANAGED_RULE_REF_PREFIX) + ] + return [*unmanaged, *[_portable_rule(rule) for rule in managed_rules]] + + +class CloudflareApi: + def __init__(self, token: str): + self.token = token + + def request( + self, + method: str, + path: str, + payload: Dict[str, Any] | None = None, + *, + allow_not_found: bool = False, + ) -> Dict[str, Any] | None: + body = None if payload is None else json.dumps(payload).encode("utf-8") + request = Request( + f"{API_BASE}{path}", + data=body, + method=method, + headers={ + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "User-Agent": "polyweather-cloudflare-config/1.0", + }, + ) + try: + with urlopen(request, timeout=30) as response: + result = json.loads(response.read().decode("utf-8")) + except HTTPError as exc: + if allow_not_found and exc.code == 404: + return None + detail = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Cloudflare API {method} {path} failed: {exc.code} {detail}") from exc + if not result.get("success"): + raise RuntimeError(f"Cloudflare API {method} {path} failed: {result.get('errors')}") + return result + + +def resolve_zone_id(api: CloudflareApi, zone_name: str, explicit_zone_id: str) -> str: + if explicit_zone_id: + return explicit_zone_id + result = api.request("GET", f"/zones?{urlencode({'name': zone_name, 'status': 'active'})}") + zones = list((result or {}).get("result") or []) + if len(zones) != 1: + raise RuntimeError(f"Expected one active Cloudflare zone named {zone_name}, found {len(zones)}") + return str(zones[0]["id"]) + + +def apply_cache_rules(api: CloudflareApi, zone_id: str) -> List[Dict[str, Any]]: + path = f"/zones/{zone_id}/rulesets/phases/{PHASE}/entrypoint" + current = api.request("GET", path, allow_not_found=True) or {} + existing_rules = list((current.get("result") or {}).get("rules") or []) + merged_rules = merge_managed_rules(existing_rules, build_managed_cache_rules()) + api.request( + "PUT", + path, + { + "description": "PolyWeather zone-level cache rules", + "rules": merged_rules, + }, + ) + return merged_rules + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Apply rules to Cloudflare") + parser.add_argument("--zone-name", default="polyweather.top") + parser.add_argument("--zone-id", default=os.getenv("CLOUDFLARE_ZONE_ID", "")) + parser.add_argument("--api-token", default=os.getenv("CLOUDFLARE_API_TOKEN", "")) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + managed_rules = build_managed_cache_rules() + if not args.apply: + print(json.dumps({"mode": "plan", "managed_rules": managed_rules}, indent=2)) + return 0 + if not args.api_token: + print("CLOUDFLARE_API_TOKEN or --api-token is required with --apply", file=sys.stderr) + return 2 + api = CloudflareApi(args.api_token) + zone_id = resolve_zone_id(api, args.zone_name, args.zone_id) + merged_rules = apply_cache_rules(api, zone_id) + print( + json.dumps( + { + "mode": "applied", + "zone_id": zone_id, + "managed_rule_count": len(managed_rules), + "total_rule_count": len(merged_rules), + }, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cloudflare_free_config.py b/tests/test_cloudflare_free_config.py new file mode 100644 index 00000000..a6fbe821 --- /dev/null +++ b/tests/test_cloudflare_free_config.py @@ -0,0 +1,48 @@ +from scripts.configure_cloudflare_free import ( + MANAGED_RULE_REF_PREFIX, + build_managed_cache_rules, + merge_managed_rules, +) + + +def test_cloudflare_managed_rules_cache_public_content_then_bypass_sensitive_requests(): + rules = build_managed_cache_rules() + + assert [rule["ref"] for rule in rules] == [ + f"{MANAGED_RULE_REF_PREFIX}public", + f"{MANAGED_RULE_REF_PREFIX}bypass", + ] + assert rules[0]["action_parameters"]["cache"] is True + assert rules[-1]["action_parameters"]["cache"] is False + assert 'http.host eq "api.polyweather.top"' in rules[-1]["expression"] + assert 'http.request.uri.query contains "force_refresh=true"' in rules[-1]["expression"] + + +def test_cloudflare_rule_merge_preserves_unmanaged_rules_and_puts_bypass_last(): + existing = [ + { + "ref": "existing_rule", + "description": "keep me", + "expression": 'http.host eq "example.com"', + "action": "set_cache_settings", + "action_parameters": {"cache": True}, + "enabled": True, + }, + { + "ref": f"{MANAGED_RULE_REF_PREFIX}old", + "description": "replace me", + "expression": "true", + "action": "set_cache_settings", + "action_parameters": {"cache": False}, + "enabled": True, + }, + ] + + merged = merge_managed_rules(existing, build_managed_cache_rules()) + + assert [rule["ref"] for rule in merged] == [ + "existing_rule", + f"{MANAGED_RULE_REF_PREFIX}public", + f"{MANAGED_RULE_REF_PREFIX}bypass", + ] + assert merged[-1]["action_parameters"]["cache"] is False diff --git a/tests/test_deployment_runtime_config.py b/tests/test_deployment_runtime_config.py index 80e450b9..e450c3a6 100644 --- a/tests/test_deployment_runtime_config.py +++ b/tests/test_deployment_runtime_config.py @@ -138,6 +138,13 @@ def test_scan_terminal_backend_timeout_returns_before_next_proxy_abort(): ) +def test_deploy_workflow_applies_cloudflare_rules_when_token_is_available(): + workflow = (ROOT / ".github" / "workflows" / "ci.yml").read_text(encoding="utf-8") + + assert "CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}" in workflow + assert "python scripts/configure_cloudflare_free.py --apply" in workflow + + def test_probability_engine_uses_enriched_multi_model_snapshot(): source = (ROOT / "web" / "analysis_service.py").read_text(encoding="utf-8") diff --git a/tests/test_scan_terminal_modules.py b/tests/test_scan_terminal_modules.py index b273033f..4a2ac38b 100644 --- a/tests/test_scan_terminal_modules.py +++ b/tests/test_scan_terminal_modules.py @@ -10,6 +10,7 @@ from web.scan_terminal_payloads import ( SCAN_PAYLOAD_FULL_RUNWAY_HISTORY_ROWS, ) from web.scan_terminal_ranker import build_ranked_scan_terminal_result +from web import scan_terminal_city_row from web.scan_terminal_city_row import _build_quick_row from web.routers.scan import router as scan_router from web.scan_terminal_service import _scan_terminal_prewarm_filters @@ -81,6 +82,43 @@ def test_scan_terminal_prewarm_covers_default_api_limit(): assert 180 in limits +def test_scan_city_terminal_rows_reuses_persisted_panel_cache(monkeypatch): + payload = { + "display_name": "Paris", + "local_date": "2026-06-10", + "local_time": "12:00", + "current": {"temp": 18.0}, + "risk": {}, + "deb": {"prediction": 20.0}, + "probabilities": {}, + "multi_model": {}, + } + + class _Cache: + @staticmethod + def get_city_cache(kind, city): + assert (kind, city) == ("panel", "paris") + return {"payload": payload} + + monkeypatch.setattr(scan_terminal_city_row, "_PANEL_CACHE_DB", _Cache()) + monkeypatch.setattr( + scan_terminal_city_row, + "_analyze", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("non-force scan must not fetch external sources") + ), + ) + + result = scan_terminal_city_row._scan_city_terminal_rows( + "paris", + {"market_type": "maxtemp"}, + force_refresh=False, + ) + + assert result["city"] == "paris" + assert result["rows"][0]["current_temp"] == 18.0 + + def test_scan_router_does_not_expose_terminal_ai_endpoint(): routes = { getattr(route, "path", None): getattr(route, "methods", set()) diff --git a/tests/test_web_observability.py b/tests/test_web_observability.py index 1d773167..1bf64831 100644 --- a/tests/test_web_observability.py +++ b/tests/test_web_observability.py @@ -3398,6 +3398,41 @@ def test_scan_terminal_cold_requests_start_background_build_without_blocking(mon assert all("初始化" in result["stale_reason"] or "刷新中" in result["stale_reason"] for result in results) +def test_scan_terminal_background_refresh_reuses_cached_city_data(monkeypatch): + filters = {"scan_mode": "tradable", "limit": 17, "min_edge_pct": 6.75} + calls = [] + + monkeypatch.setattr( + scan_terminal_service, + "mark_scan_terminal_refreshing", + lambda _filters: True, + ) + monkeypatch.setattr( + scan_terminal_service, + "clear_scan_terminal_refreshing", + lambda _filters: None, + ) + monkeypatch.setattr( + scan_terminal_service, + "_build_scan_terminal_payload_singleflight", + lambda filters_arg, *, force_refresh=False: calls.append( + (dict(filters_arg), force_refresh) + ), + ) + + class _ImmediateThread: + def __init__(self, *, target, name, daemon): + self._target = target + + def start(self): + self._target() + + monkeypatch.setattr(scan_terminal_service.threading, "Thread", _ImmediateThread) + + assert scan_terminal_service._start_scan_terminal_background_refresh(filters) is True + assert calls == [(filters, False)] + + def test_scan_terminal_nonforce_ignores_ancient_success_snapshot(monkeypatch): filters = {"scan_mode": "tradable", "limit": 17, "min_edge_pct": 6.75} old_success_t = 1780839484.0 diff --git a/web/scan_terminal_city_row.py b/web/scan_terminal_city_row.py index 28c7bc40..3a0f95a4 100644 --- a/web/scan_terminal_city_row.py +++ b/web/scan_terminal_city_row.py @@ -5,6 +5,7 @@ import re from datetime import datetime, timedelta from typing import Any, Dict, List, Optional +from src.database.db_manager import DBManager from web.core import CITIES, _sf as _safe_float from web.analysis_service import _analyze from web.scan_terminal_filters import ( @@ -16,6 +17,7 @@ from web.services.city_payloads import aggregate_runway_history SCAN_ROW_RUNWAY_HISTORY_RESOLUTION = "10m" SCAN_ROW_MAX_RUNWAY_POINTS = 144 +_PANEL_CACHE_DB = DBManager() def _compact_runway_plate_history_for_scan(raw_history: Any) -> Dict[str, List[Dict[str, Any]]]: @@ -171,11 +173,18 @@ def _scan_city_terminal_rows_quick( ) -> Dict[str, Any]: """Fast path that returns cached analysis rows only — returns a single row per city with cached analysis data (Obs, DEB, probabilities) but no market prices.""" - data = _analyze( - city, - force_refresh=force_refresh, - detail_mode="panel", - ) + data: Dict[str, Any] = {} + if not force_refresh: + cached_entry = _PANEL_CACHE_DB.get_city_cache("panel", city) + cached_payload = cached_entry.get("payload") if isinstance(cached_entry, dict) else None + if isinstance(cached_payload, dict): + data = cached_payload + if not data: + data = _analyze( + city, + force_refresh=force_refresh, + detail_mode="panel", + ) row = _build_quick_row(city=city, data=data) return { "city": city, diff --git a/web/scan_terminal_service.py b/web/scan_terminal_service.py index 9420473d..430d210e 100644 --- a/web/scan_terminal_service.py +++ b/web/scan_terminal_service.py @@ -104,7 +104,7 @@ def _start_scan_terminal_background_refresh(filters: Dict[str, Any]) -> bool: def _runner() -> None: try: - _build_scan_terminal_payload_singleflight(filters, force_refresh=True) + _build_scan_terminal_payload_singleflight(filters, force_refresh=False) except Exception as exc: # pragma: no cover - defensive background guard logger.warning("scan terminal background refresh failed: {}", exc) finally: