Optimize multi-model caching and frontend revalidation
This commit is contained in:
@@ -860,7 +860,7 @@ export function DashboardStoreProvider({
|
|||||||
void (async () => {
|
void (async () => {
|
||||||
try {
|
try {
|
||||||
const latestDetail = await dashboardClient.getCityDetail(cityName, {
|
const latestDetail = await dashboardClient.getCityDetail(cityName, {
|
||||||
force: true,
|
force: false,
|
||||||
depth,
|
depth,
|
||||||
});
|
});
|
||||||
const detail = latestDetail;
|
const detail = latestDetail;
|
||||||
|
|||||||
@@ -518,8 +518,8 @@ export const dashboardClient = {
|
|||||||
meta: Record<string, CityCacheMeta>,
|
meta: Record<string, CityCacheMeta>,
|
||||||
) {
|
) {
|
||||||
if (!isClient()) return;
|
if (!isClient()) return;
|
||||||
// Keep only the 3 most-recently-accessed cities to prevent sessionStorage bloat
|
// Keep only the 12 most-recently-accessed cities to prevent sessionStorage bloat
|
||||||
const MAX_CACHED_CITIES = 3;
|
const MAX_CACHED_CITIES = 12;
|
||||||
const allEntries = Object.entries(details).map(([cityName, detail]) => ({
|
const allEntries = Object.entries(details).map(([cityName, detail]) => ({
|
||||||
cityName,
|
cityName,
|
||||||
cachedAt: meta[cityName]?.cachedAt || 0,
|
cachedAt: meta[cityName]?.cachedAt || 0,
|
||||||
|
|||||||
@@ -307,6 +307,16 @@ class DBManager:
|
|||||||
source_fingerprint TEXT
|
source_fingerprint TEXT
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
|
conn.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS city_full_cache (
|
||||||
|
city TEXT PRIMARY KEY,
|
||||||
|
payload_json TEXT NOT NULL,
|
||||||
|
updated_at TEXT NOT NULL,
|
||||||
|
updated_at_ts REAL NOT NULL,
|
||||||
|
version TEXT,
|
||||||
|
source_fingerprint TEXT
|
||||||
|
)
|
||||||
|
""")
|
||||||
conn.execute("""
|
conn.execute("""
|
||||||
CREATE TABLE IF NOT EXISTS cache_refresh_locks (
|
CREATE TABLE IF NOT EXISTS cache_refresh_locks (
|
||||||
cache_key TEXT PRIMARY KEY,
|
cache_key TEXT PRIMARY KEY,
|
||||||
@@ -432,6 +442,8 @@ class DBManager:
|
|||||||
return "city_nearby_cache"
|
return "city_nearby_cache"
|
||||||
if normalized == "market":
|
if normalized == "market":
|
||||||
return "city_market_cache"
|
return "city_market_cache"
|
||||||
|
if normalized == "full":
|
||||||
|
return "city_full_cache"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_city_cache(self, kind: str, city: str) -> Optional[Dict[str, Any]]:
|
def get_city_cache(self, kind: str, city: str) -> Optional[Dict[str, Any]]:
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import tempfile
|
||||||
|
import os
|
||||||
|
from src.database.db_manager import DBManager
|
||||||
|
|
||||||
|
def test_full_cache_lifecycle():
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
db_path = os.path.join(tmpdir, "test.db")
|
||||||
|
if hasattr(DBManager, "_initialized_paths"):
|
||||||
|
DBManager._initialized_paths.clear()
|
||||||
|
|
||||||
|
db = DBManager(db_path)
|
||||||
|
|
||||||
|
# Verify table name resolution
|
||||||
|
assert db._cache_table_name("full") == "city_full_cache"
|
||||||
|
|
||||||
|
# Verify cache save and load lifecycle
|
||||||
|
city = "testcity"
|
||||||
|
payload = {"data": "test_payload_full_depth", "nested": {"key": 123}}
|
||||||
|
|
||||||
|
db.set_city_cache(
|
||||||
|
kind="full",
|
||||||
|
city=city,
|
||||||
|
payload=payload,
|
||||||
|
version="v1",
|
||||||
|
source_fingerprint="testcity:full"
|
||||||
|
)
|
||||||
|
|
||||||
|
cached = db.get_city_cache("full", city)
|
||||||
|
assert cached is not None
|
||||||
|
assert cached["city"] == city
|
||||||
|
assert cached["payload"] == payload
|
||||||
|
assert cached["version"] == "v1"
|
||||||
|
assert cached["source_fingerprint"] == "testcity:full"
|
||||||
@@ -79,6 +79,15 @@ async def get_city_detail_payload(
|
|||||||
detail_mode = "nearby"
|
detail_mode = "nearby"
|
||||||
else:
|
else:
|
||||||
detail_mode = "panel"
|
detail_mode = "panel"
|
||||||
|
if detail_mode == "full":
|
||||||
|
if force_refresh:
|
||||||
|
return await run_in_threadpool(legacy_routes._refresh_city_full_cache, city, True)
|
||||||
|
cached_entry = await run_in_threadpool(legacy_routes._CACHE_DB.get_city_cache, "full", city)
|
||||||
|
if cached_entry:
|
||||||
|
if not legacy_routes._city_cache_is_fresh(cached_entry, legacy_routes.CITY_FULL_CACHE_TTL_SEC):
|
||||||
|
return await run_in_threadpool(legacy_routes._refresh_city_full_cache, city, False)
|
||||||
|
return cached_entry.get("payload") or {}
|
||||||
|
return await run_in_threadpool(legacy_routes._refresh_city_full_cache, city, False)
|
||||||
if detail_mode == "panel":
|
if detail_mode == "panel":
|
||||||
if force_refresh:
|
if force_refresh:
|
||||||
return await run_in_threadpool(legacy_routes._refresh_city_panel_cache, city, True)
|
return await run_in_threadpool(legacy_routes._refresh_city_panel_cache, city, True)
|
||||||
|
|||||||
@@ -142,6 +142,7 @@ CITY_SUMMARY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_SUMMARY_CAC
|
|||||||
CITY_PANEL_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_PANEL_CACHE_TTL_SEC", "1800")))
|
CITY_PANEL_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_PANEL_CACHE_TTL_SEC", "1800")))
|
||||||
CITY_NEARBY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_NEARBY_CACHE_TTL_SEC", "1800")))
|
CITY_NEARBY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_NEARBY_CACHE_TTL_SEC", "1800")))
|
||||||
CITY_MARKET_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_MARKET_CACHE_TTL_SEC", "1800")))
|
CITY_MARKET_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_MARKET_CACHE_TTL_SEC", "1800")))
|
||||||
|
CITY_FULL_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_FULL_CACHE_TTL_SEC", "1800")))
|
||||||
MARKET_SCAN_PAYLOAD_TTL_SEC = max(
|
MARKET_SCAN_PAYLOAD_TTL_SEC = max(
|
||||||
5,
|
5,
|
||||||
int(os.getenv("POLYWEATHER_MARKET_SCAN_PAYLOAD_TTL_SEC", "30")),
|
int(os.getenv("POLYWEATHER_MARKET_SCAN_PAYLOAD_TTL_SEC", "30")),
|
||||||
@@ -319,6 +320,18 @@ def _refresh_city_market_cache(city: str, force_refresh: bool = False) -> dict:
|
|||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _refresh_city_full_cache(city: str, force_refresh: bool = False) -> dict:
|
||||||
|
payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=True, detail_mode="full")
|
||||||
|
_CACHE_DB.set_city_cache(
|
||||||
|
"full",
|
||||||
|
city,
|
||||||
|
payload,
|
||||||
|
version="v1",
|
||||||
|
source_fingerprint=f"{city}:full",
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def _schedule_cache_refresh(
|
def _schedule_cache_refresh(
|
||||||
background_tasks: BackgroundTasks,
|
background_tasks: BackgroundTasks,
|
||||||
@@ -329,7 +342,7 @@ def _schedule_cache_refresh(
|
|||||||
) -> bool:
|
) -> bool:
|
||||||
normalized_kind = str(kind or "").strip().lower()
|
normalized_kind = str(kind or "").strip().lower()
|
||||||
normalized_city = str(city or "").strip().lower()
|
normalized_city = str(city or "").strip().lower()
|
||||||
if normalized_kind not in {"summary", "panel", "nearby", "market"} or not normalized_city:
|
if normalized_kind not in {"summary", "panel", "nearby", "market", "full"} or not normalized_city:
|
||||||
return False
|
return False
|
||||||
cache_key = f"city:{normalized_kind}:{normalized_city}"
|
cache_key = f"city:{normalized_kind}:{normalized_city}"
|
||||||
owner = _CACHE_DB.acquire_cache_refresh_lock(
|
owner = _CACHE_DB.acquire_cache_refresh_lock(
|
||||||
@@ -347,8 +360,10 @@ def _schedule_cache_refresh(
|
|||||||
_refresh_city_panel_cache(normalized_city, force_refresh=force_refresh)
|
_refresh_city_panel_cache(normalized_city, force_refresh=force_refresh)
|
||||||
elif normalized_kind == "nearby":
|
elif normalized_kind == "nearby":
|
||||||
_refresh_city_nearby_cache(normalized_city, force_refresh=force_refresh)
|
_refresh_city_nearby_cache(normalized_city, force_refresh=force_refresh)
|
||||||
else:
|
elif normalized_kind == "market":
|
||||||
_refresh_city_market_cache(normalized_city, force_refresh=force_refresh)
|
_refresh_city_market_cache(normalized_city, force_refresh=force_refresh)
|
||||||
|
else:
|
||||||
|
_refresh_city_full_cache(normalized_city, force_refresh=force_refresh)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"cache refresh failed kind={} city={} force_refresh={}: {}",
|
"cache refresh failed kind={} city={} force_refresh={}: {}",
|
||||||
|
|||||||
@@ -35,7 +35,9 @@ def get_system_cache_status(request: Request, cities: Optional[str] = None) -> D
|
|||||||
"summary": legacy_routes.CITY_SUMMARY_CACHE_TTL_SEC,
|
"summary": legacy_routes.CITY_SUMMARY_CACHE_TTL_SEC,
|
||||||
"panel": legacy_routes.CITY_PANEL_CACHE_TTL_SEC,
|
"panel": legacy_routes.CITY_PANEL_CACHE_TTL_SEC,
|
||||||
"nearby": legacy_routes.CITY_NEARBY_CACHE_TTL_SEC,
|
"nearby": legacy_routes.CITY_NEARBY_CACHE_TTL_SEC,
|
||||||
"market": legacy_routes.CITY_MARKET_CACHE_TTL_SEC, }
|
"market": legacy_routes.CITY_MARKET_CACHE_TTL_SEC,
|
||||||
|
"full": legacy_routes.CITY_FULL_CACHE_TTL_SEC,
|
||||||
|
}
|
||||||
items = []
|
items = []
|
||||||
for city in selected:
|
for city in selected:
|
||||||
row = {"city": city}
|
row = {"city": city}
|
||||||
@@ -71,6 +73,7 @@ def run_system_priority_warm(
|
|||||||
legacy_routes._refresh_city_panel_cache(city, force_refresh=False)
|
legacy_routes._refresh_city_panel_cache(city, force_refresh=False)
|
||||||
legacy_routes._refresh_city_nearby_cache(city, force_refresh=False)
|
legacy_routes._refresh_city_nearby_cache(city, force_refresh=False)
|
||||||
legacy_routes._refresh_city_market_cache(city, force_refresh=False)
|
legacy_routes._refresh_city_market_cache(city, force_refresh=False)
|
||||||
|
legacy_routes._refresh_city_full_cache(city, force_refresh=False)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.warning("priority warm primary failed city={} timezone={}: {}", city, timezone, exc)
|
logger.warning("priority warm primary failed city={} timezone={}: {}", city, timezone, exc)
|
||||||
for city in secondary:
|
for city in secondary:
|
||||||
|
|||||||
Reference in New Issue
Block a user