gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stop collector from triggering full weather refreshes

Browse Source
This commit is contained in:
2569718930@qq.com
2026-06-14 18:44:49 +08:00
parent 3e1e06d8e1
commit ccc8866229
9 changed files with 180 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/services/city_api.py
+11 -107
View File
@@ -7,7 +7,7 @@ import asyncio
import threading
import time
from copy import deepcopy
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
from typing import Any, Callable, Dict, List, Optional, Tuple
from fastapi import HTTPException, Request
from fastapi.concurrency import run_in_threadpool
@@ -26,12 +26,6 @@ _RECENT_DEB_CACHE_TTL_SEC = max(
60,
int(os.getenv("POLYWEATHER_CITIES_DEB_RECENT_CACHE_TTL_SEC", "300") or "300"),
)
_CITY_FULL_REFRESH_INFLIGHT: Dict[str, "asyncio.Task[Dict[str, Any]]"] = {}
_CITY_FULL_STALE_REFRESH_TASKS: Dict[str, "asyncio.Task[Dict[str, Any]]"] = {}
_CITY_FULL_REFRESH_LOCK = asyncio.Lock()
_CITY_FORCE_REFRESH_INFLIGHT: Dict[str, "asyncio.Task[Dict[str, Any]]"] = {}
_CITY_FORCE_REFRESH_LOCK = asyncio.Lock()
_CITY_STALE_REFRESH_TASKS: Dict[str, "asyncio.Task[Dict[str, Any]]"] = {}
CityDetailPayloadCacheKey = Tuple[str, str, str, str, str, int]
CityChartDetailPayloadCacheKey = Tuple[str, str, str, int]
CityDetailBatchResponseCacheKey = Tuple[Tuple[str, ...], bool, str, str, str, str]
@@ -71,14 +65,6 @@ def _city_detail_batch_response_cache_ttl() -> float:
return max(0.0, min(30.0, value))
def _city_force_refresh_timeout_sec() -> float:
try:
value = float(os.getenv("POLYWEATHER_CITY_FORCE_REFRESH_TIMEOUT_SEC", "8") or "8")
except ValueError:
value = 8.0
return max(0.01, min(30.0, value))
def _city_chart_optional_overlay_timeout_sec() -> float:
try:
timeout_ms = int(
@@ -193,7 +179,6 @@ def _enqueue_collector_refresh_request(
def _request_city_cache_refresh(
city: str,
kind: str,
refresh_fn: Callable[[str, bool], Dict[str, Any]],
) -> None:
_enqueue_collector_refresh_request(city, kind)
@@ -252,34 +237,9 @@ def _queue_and_build_initializing_city_payload(city: str, *, kind: str) -> Dict[
return _build_initializing_city_payload(city, detail_depth=kind)
async def _get_or_start_city_force_refresh_task(
key: str,
refresh_factory: Callable[[], Awaitable[Dict[str, Any]]],
) -> Tuple["asyncio.Task[Dict[str, Any]]", bool]:
async with _CITY_FORCE_REFRESH_LOCK:
task = _CITY_FORCE_REFRESH_INFLIGHT.get(key)
started = False
if task is None or task.done():
task = asyncio.create_task(refresh_factory())
_CITY_FORCE_REFRESH_INFLIGHT[key] = task
started = True
def _cleanup(done: "asyncio.Task[Dict[str, Any]]") -> None:
if _CITY_FORCE_REFRESH_INFLIGHT.get(key) is done:
_CITY_FORCE_REFRESH_INFLIGHT.pop(key, None)
try:
done.result()
except Exception as exc: # pragma: no cover - defensive background guard
logger.warning("city force refresh failed key={}: {}", key, exc)
task.add_done_callback(_cleanup)
return task, started
async def _refresh_city_payload_with_stale_timeout(
city: str,
kind: str,
refresh_factory: Callable[[], Awaitable[Dict[str, Any]]],
) -> Dict[str, Any]:
cached_before_refresh = await _get_cached_city_payload(city, kind)
if not cached_before_refresh:
@@ -306,19 +266,16 @@ async def _refresh_city_payload_with_stale_timeout(
async def _refresh_city_cache_with_stale_timeout(
city: str,
kind: str,
refresh_fn: Callable[[str, bool], Dict[str, Any]],
) -> Dict[str, Any]:
return await _refresh_city_payload_with_stale_timeout(
city,
kind,
lambda: run_in_threadpool(refresh_fn, city, True),
)
def _start_city_cache_stale_refresh(
city: str,
kind: str,
refresh_fn: Callable[[str, bool], Dict[str, Any]],
) -> None:
normalized = str(city or "").strip().lower()
cache_kind = str(kind or "").strip().lower()
@@ -394,54 +351,6 @@ def _overlay_cached_runway_history_from_db(city: str, payload: Dict[str, Any]) -
return next_payload
async def _refresh_city_full_cache_singleflight(city: str, force_refresh: bool) -> Dict[str, Any]:
key = f"{city}:{bool(force_refresh)}"
async with _CITY_FULL_REFRESH_LOCK:
task = _CITY_FULL_REFRESH_INFLIGHT.get(key)
if task is None:
async def _run_refresh() -> Dict[str, Any]:
try:
return await run_in_threadpool(
legacy_routes._refresh_city_full_cache,
city,
force_refresh,
)
finally:
await _invalidate_city_detail_payload_cache(city)
task = asyncio.create_task(_run_refresh())
_CITY_FULL_REFRESH_INFLIGHT[key] = task
try:
return await task
finally:
if task.done():
async with _CITY_FULL_REFRESH_LOCK:
if _CITY_FULL_REFRESH_INFLIGHT.get(key) is task:
_CITY_FULL_REFRESH_INFLIGHT.pop(key, None)
async def _invalidate_city_detail_payload_cache(city: str) -> None:
normalized = str(city or "").strip().lower()
if not normalized:
return
async with _CITY_DETAIL_PAYLOAD_LOCK:
_CITY_DETAIL_PAYLOAD_EPOCH[normalized] = _CITY_DETAIL_PAYLOAD_EPOCH.get(normalized, 0) + 1
old_keys = [key for key in _CITY_DETAIL_PAYLOAD_CACHE if key[0] == normalized]
for key in old_keys:
_CITY_DETAIL_PAYLOAD_CACHE.pop(key, None)
_CITY_DETAIL_PAYLOAD_CACHE_TS.pop(key, None)
async with _CITY_CHART_DETAIL_PAYLOAD_LOCK:
old_chart_keys = [key for key in _CITY_CHART_DETAIL_PAYLOAD_CACHE if key[0] == normalized]
for key in old_chart_keys:
_CITY_CHART_DETAIL_PAYLOAD_CACHE.pop(key, None)
_CITY_CHART_DETAIL_PAYLOAD_CACHE_TS.pop(key, None)
async def _refresh_city_full_data(city: str, force_refresh: bool) -> Dict[str, Any]:
await _invalidate_city_detail_payload_cache(city)
return await _refresh_city_full_cache_singleflight(city, force_refresh)
def _start_city_full_stale_refresh(city: str) -> None:
normalized = str(city or "").strip().lower()
if not normalized:
@@ -454,7 +363,6 @@ async def _get_city_full_data(city: str, *, force_refresh: bool) -> Dict[str, An
return await _refresh_city_payload_with_stale_timeout(
city,
"full",
lambda: _refresh_city_full_data(city, True),
)
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "full", city)
if cached_entry:
@@ -792,20 +700,19 @@ async def get_city_detail_payload(
return await _refresh_city_cache_with_stale_timeout(
city,
"panel",
legacy_routes._refresh_city_panel_cache,
)
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "panel", city)
if cached_entry:
if not legacy_routes._city_cache_is_fresh(cached_entry, legacy_routes.CITY_PANEL_CACHE_TTL_SEC):
payload = cached_entry.get("payload") or {}
if payload:
_start_city_cache_stale_refresh(city, "panel", legacy_routes._refresh_city_panel_cache)
_start_city_cache_stale_refresh(city, "panel")
return await _overlay_cached_wunderground(city, payload)
return _queue_and_build_initializing_city_payload(city, kind="panel")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
canonical_payload = await _get_canonical_city_payload(city, detail_depth="panel")
if canonical_payload:
_request_city_cache_refresh(city, "panel", legacy_routes._refresh_city_panel_cache)
_request_city_cache_refresh(city, "panel")
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="panel")
if detail_mode == "nearby":
@@ -813,24 +720,23 @@ async def get_city_detail_payload(
return await _refresh_city_cache_with_stale_timeout(
city,
"nearby",
legacy_routes._refresh_city_nearby_cache,
)
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "nearby", city)
if cached_entry:
if not legacy_routes._city_cache_is_fresh(cached_entry, legacy_routes.CITY_NEARBY_CACHE_TTL_SEC):
payload = cached_entry.get("payload") or {}
if payload:
_start_city_cache_stale_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
_start_city_cache_stale_refresh(city, "nearby")
return await _overlay_cached_wunderground(city, payload)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="nearby")
if canonical_payload:
_request_city_cache_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
_request_city_cache_refresh(city, "nearby")
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="nearby")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
canonical_payload = await _get_canonical_city_payload(city, detail_depth="nearby")
if canonical_payload:
_request_city_cache_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
_request_city_cache_refresh(city, "nearby")
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="nearby")
if detail_mode == "market":
@@ -838,24 +744,23 @@ async def get_city_detail_payload(
return await _refresh_city_cache_with_stale_timeout(
city,
"market",
legacy_routes._refresh_city_market_cache,
)
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "market", city)
if cached_entry:
if not legacy_routes._market_analysis_cache_is_fresh(cached_entry):
payload = cached_entry.get("payload") or {}
if payload:
_start_city_cache_stale_refresh(city, "market", legacy_routes._refresh_city_market_cache)
_start_city_cache_stale_refresh(city, "market")
return await _overlay_cached_wunderground(city, payload)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="market")
if canonical_payload:
_request_city_cache_refresh(city, "market", legacy_routes._refresh_city_market_cache)
_request_city_cache_refresh(city, "market")
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="market")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
canonical_payload = await _get_canonical_city_payload(city, detail_depth="market")
if canonical_payload:
_request_city_cache_refresh(city, "market", legacy_routes._refresh_city_market_cache)
_request_city_cache_refresh(city, "market")
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="market")
return await run_in_threadpool(legacy_routes._analyze, city, force_refresh, False, detail_mode)
@@ -872,20 +777,19 @@ async def get_city_summary_payload(
return await _refresh_city_cache_with_stale_timeout(
city,
"summary",
legacy_routes._refresh_city_summary_cache,
)
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "summary", city)
if cached_entry:
if not legacy_routes._city_cache_is_fresh(cached_entry, legacy_routes.CITY_SUMMARY_CACHE_TTL_SEC):
payload = cached_entry.get("payload") or {}
if payload:
_start_city_cache_stale_refresh(city, "summary", legacy_routes._refresh_city_summary_cache)
_start_city_cache_stale_refresh(city, "summary")
return await _overlay_cached_wunderground(city, payload)
return _queue_and_build_initializing_city_payload(city, kind="summary")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
canonical_payload = await _get_canonical_city_payload(city, detail_depth="summary")
if canonical_payload:
_request_city_cache_refresh(city, "summary", legacy_routes._refresh_city_summary_cache)
_request_city_cache_refresh(city, "summary")
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="summary")