Add cached dashboard prewarm hints for priority cities

This commit is contained in:
2569718930@qq.com
2026-04-16 15:05:14 +08:00
parent b635070106
commit c0e5307893
5 changed files with 672 additions and 123 deletions
+19
View File
@@ -618,6 +618,25 @@ export function DashboardStoreProvider({
void loadCities(); void loadCities();
}, []); }, []);
useEffect(() => {
if (!cities.length) return;
if (typeof window === "undefined") return;
const schedule = () => dashboardClient.sendPriorityWarmHint();
const idleCallback = (window as Window & {
requestIdleCallback?: (callback: () => void, options?: { timeout: number }) => number;
}).requestIdleCallback;
if (typeof idleCallback === "function") {
const id = idleCallback(schedule, { timeout: 3000 });
return () => {
if (typeof window.cancelIdleCallback === "function") {
window.cancelIdleCallback(id);
}
};
}
const timer = window.setTimeout(schedule, 2000);
return () => window.clearTimeout(timer);
}, [cities.length]);
useEffect(() => { useEffect(() => {
void refreshProAccess(); void refreshProAccess();
}, []); }, []);
+19
View File
@@ -12,6 +12,7 @@ const CACHE_TTL_MS = 5 * 60 * 1000;
const pendingCityDetailRequests = new Map<string, Promise<CityDetail>>(); const pendingCityDetailRequests = new Map<string, Promise<CityDetail>>();
const pendingHistoryRequests = new Map<string, Promise<HistoryPayload>>(); const pendingHistoryRequests = new Map<string, Promise<HistoryPayload>>();
const pendingCitySummaryRequests = new Map<string, Promise<CitySummary>>(); const pendingCitySummaryRequests = new Map<string, Promise<CitySummary>>();
const PRIORITY_WARM_SESSION_KEY = "polyWeather_priority_warm_v1";
type CityCacheMeta = { type CityCacheMeta = {
cachedAt: number; cachedAt: number;
@@ -142,6 +143,24 @@ export const dashboardClient = {
return data.cities || []; return data.cities || [];
}, },
sendPriorityWarmHint(timezone?: string | null) {
if (!isClient()) return;
const tz = String(
timezone || Intl.DateTimeFormat().resolvedOptions().timeZone || "",
).trim();
if (!tz) return;
const cacheKey = `${PRIORITY_WARM_SESSION_KEY}:${tz}`;
if (window.sessionStorage.getItem(cacheKey)) return;
window.sessionStorage.setItem(cacheKey, "1");
const params = new URLSearchParams({ timezone: tz });
void fetch(`/api/system/priority-warm?${params.toString()}`, {
method: "POST",
headers: { Accept: "application/json" },
cache: "no-store",
keepalive: true,
}).catch(() => {});
},
async getCitySummary(cityName: string, options?: { force?: boolean }) { async getCitySummary(cityName: string, options?: { force?: boolean }) {
const force = options?.force ?? false; const force = options?.force ?? false;
const requestKey = `${cityName}::${force ? "force" : "cached"}`; const requestKey = `${cityName}::${force ? "force" : "cached"}`;
+203
View File
@@ -182,6 +182,64 @@ class DBManager:
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
) )
""") """)
conn.execute("""
CREATE TABLE IF NOT EXISTS city_summary_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_panel_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_nearby_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_market_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_history_preview_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS cache_refresh_locks (
cache_key TEXT PRIMARY KEY,
locked_until_ts REAL NOT NULL,
owner TEXT NOT NULL,
updated_at TEXT NOT NULL
)
""")
conn.execute(""" conn.execute("""
CREATE TABLE IF NOT EXISTS payment_audit_events ( CREATE TABLE IF NOT EXISTS payment_audit_events (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -247,6 +305,151 @@ class DBManager:
conn.commit() conn.commit()
logger.info(f"Database initialized successfully path={self.db_path}") logger.info(f"Database initialized successfully path={self.db_path}")
def _cache_table_name(self, kind: str) -> Optional[str]:
normalized = str(kind or "").strip().lower()
if normalized == "summary":
return "city_summary_cache"
if normalized == "panel":
return "city_panel_cache"
if normalized == "nearby":
return "city_nearby_cache"
if normalized == "market":
return "city_market_cache"
if normalized == "history_preview":
return "city_history_preview_cache"
return None
def get_city_cache(self, kind: str, city: str) -> Optional[Dict[str, Any]]:
table = self._cache_table_name(kind)
normalized_city = str(city or "").strip().lower()
if not table or not normalized_city:
return None
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
row = conn.execute(
f"""
SELECT city, payload_json, updated_at, updated_at_ts, version, source_fingerprint
FROM {table}
WHERE city = ?
LIMIT 1
""",
(normalized_city,),
).fetchone()
if not row:
return None
try:
payload = json.loads(str(row["payload_json"] or "{}"))
except Exception:
return None
if not isinstance(payload, dict):
return None
return {
"city": str(row["city"] or normalized_city),
"payload": payload,
"updated_at": str(row["updated_at"] or ""),
"updated_at_ts": float(row["updated_at_ts"] or 0.0),
"version": str(row["version"] or ""),
"source_fingerprint": str(row["source_fingerprint"] or ""),
}
def set_city_cache(
self,
kind: str,
city: str,
payload: Dict[str, Any],
*,
version: str = "v1",
source_fingerprint: Optional[str] = None,
) -> None:
table = self._cache_table_name(kind)
normalized_city = str(city or "").strip().lower()
if not table or not normalized_city or not isinstance(payload, dict):
return
now = datetime.now().isoformat()
now_ts = datetime.now().timestamp()
with self._get_connection() as conn:
conn.execute(
f"""
INSERT INTO {table} (
city,
payload_json,
updated_at,
updated_at_ts,
version,
source_fingerprint
)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(city) DO UPDATE SET
payload_json = excluded.payload_json,
updated_at = excluded.updated_at,
updated_at_ts = excluded.updated_at_ts,
version = excluded.version,
source_fingerprint = excluded.source_fingerprint
""",
(
normalized_city,
json.dumps(payload, ensure_ascii=False),
now,
now_ts,
str(version or "v1"),
str(source_fingerprint or ""),
),
)
conn.commit()
def acquire_cache_refresh_lock(
self,
cache_key: str,
*,
ttl_sec: int = 120,
owner: Optional[str] = None,
) -> Optional[str]:
normalized_key = str(cache_key or "").strip().lower()
if not normalized_key:
return None
lock_owner = str(owner or "").strip() or hashlib.sha1(
f"{normalized_key}:{datetime.now().timestamp()}".encode("utf-8")
).hexdigest()[:12]
now_ts = datetime.now().timestamp()
locked_until_ts = now_ts + max(15, int(ttl_sec or 120))
updated_at = datetime.now().isoformat()
with self._get_connection() as conn:
cursor = conn.execute(
"""
INSERT INTO cache_refresh_locks (cache_key, locked_until_ts, owner, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(cache_key) DO UPDATE SET
locked_until_ts = excluded.locked_until_ts,
owner = excluded.owner,
updated_at = excluded.updated_at
WHERE cache_refresh_locks.locked_until_ts < ?
""",
(
normalized_key,
locked_until_ts,
lock_owner,
updated_at,
now_ts,
),
)
conn.commit()
return lock_owner if int(cursor.rowcount or 0) > 0 else None
def release_cache_refresh_lock(self, cache_key: str, owner: str) -> None:
normalized_key = str(cache_key or "").strip().lower()
normalized_owner = str(owner or "").strip()
if not normalized_key or not normalized_owner:
return
with self._get_connection() as conn:
conn.execute(
"""
DELETE FROM cache_refresh_locks
WHERE cache_key = ? AND owner = ?
""",
(normalized_key, normalized_owner),
)
conn.commit()
def get_payment_runtime_state(self, state_key: str) -> Optional[Dict[str, Any]]: def get_payment_runtime_state(self, state_key: str) -> Optional[Dict[str, Any]]:
key = str(state_key or "").strip() key = str(state_key or "").strip()
if not key: if not key:
+2 -2
View File
@@ -208,7 +208,7 @@ def get_prewarm_runtime_summary() -> Dict[str, Any]:
"interval_sec": interval_sec, "interval_sec": interval_sec,
"jitter_sec": jitter_sec, "jitter_sec": jitter_sec,
"include_detail": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_DETAIL", True), "include_detail": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_DETAIL", True),
"include_market": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_MARKET", True), "include_market": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_MARKET", False),
"force_refresh": _truthy_env("POLYWEATHER_PREWARM_FORCE_REFRESH", False), "force_refresh": _truthy_env("POLYWEATHER_PREWARM_FORCE_REFRESH", False),
"thread_alive": bool(_WORKER_THREAD and _WORKER_THREAD.is_alive()) or shared_alive, "thread_alive": bool(_WORKER_THREAD and _WORKER_THREAD.is_alive()) or shared_alive,
"heartbeat_age_sec": None if heartbeat_age_sec is None else round(heartbeat_age_sec, 2), "heartbeat_age_sec": None if heartbeat_age_sec is None else round(heartbeat_age_sec, 2),
@@ -380,7 +380,7 @@ def start_prewarm_worker_thread() -> Optional[threading.Thread]:
jitter_sec = int(os.getenv("POLYWEATHER_PREWARM_JITTER_SEC", "20")) jitter_sec = int(os.getenv("POLYWEATHER_PREWARM_JITTER_SEC", "20"))
force_refresh = str(os.getenv("POLYWEATHER_PREWARM_FORCE_REFRESH") or "").strip().lower() in {"1", "true", "yes", "on"} force_refresh = str(os.getenv("POLYWEATHER_PREWARM_FORCE_REFRESH") or "").strip().lower() in {"1", "true", "yes", "on"}
include_detail = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_DETAIL", "true")).strip().lower() in {"1", "true", "yes", "on"} include_detail = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_DETAIL", "true")).strip().lower() in {"1", "true", "yes", "on"}
include_market = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_MARKET", "true")).strip().lower() in {"1", "true", "yes", "on"} include_market = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_MARKET", "false")).strip().lower() in {"1", "true", "yes", "on"}
thread = threading.Thread( thread = threading.Thread(
target=run_worker_loop, target=run_worker_loop,
+429 -121
View File
@@ -6,12 +6,13 @@ from datetime import datetime, timedelta
from typing import Optional from typing import Optional
from fastapi.concurrency import run_in_threadpool from fastapi.concurrency import run_in_threadpool
from fastapi import APIRouter, HTTPException, Request from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
from fastapi.responses import PlainTextResponse from fastapi.responses import PlainTextResponse
from loguru import logger from loguru import logger
from src.analysis.deb_algorithm import load_history from src.analysis.deb_algorithm import load_history
from src.analysis.probability_snapshot_archive import load_snapshot_rows_for_day from src.analysis.probability_snapshot_archive import load_snapshot_rows_for_day
from src.database.db_manager import DBManager
from src.database.runtime_state import TrainingFeatureRecordRepository, TruthRecordRepository from src.database.runtime_state import TrainingFeatureRecordRepository, TruthRecordRepository
from src.analysis.settlement_rounding import apply_city_settlement from src.analysis.settlement_rounding import apply_city_settlement
from src.data_collection.country_networks import get_country_network_provider from src.data_collection.country_networks import get_country_network_provider
@@ -55,6 +56,7 @@ from web.core import (
) )
router = APIRouter() router = APIRouter()
_CACHE_DB = DBManager()
_DEB_RECENT_LOOKBACK = 7 _DEB_RECENT_LOOKBACK = 7
_DEB_RECENT_MIN_SAMPLES = 3 _DEB_RECENT_MIN_SAMPLES = 3
@@ -91,6 +93,278 @@ DEFAULT_PREWARM_CITIES = [
"madrid", "madrid",
] ]
HISTORY_PREVIEW_DAY_LIMIT = 21 HISTORY_PREVIEW_DAY_LIMIT = 21
ASIA_CORE_CITIES = [
"hong kong",
"taipei",
"tokyo",
"seoul",
"busan",
"shanghai",
"beijing",
"guangzhou",
"shenzhen",
"chongqing",
"chengdu",
"singapore",
"kuala lumpur",
"jakarta",
]
EUROPE_CORE_CITIES = [
"istanbul",
"ankara",
"moscow",
"tel aviv",
"london",
"paris",
"madrid",
"milan",
"warsaw",
"amsterdam",
"helsinki",
]
US_CORE_CITIES = [
"new york",
"los angeles",
"san francisco",
"austin",
"houston",
"chicago",
"dallas",
"miami",
"atlanta",
"seattle",
]
CITY_SUMMARY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_SUMMARY_CACHE_TTL_SEC", "180")))
CITY_PANEL_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_PANEL_CACHE_TTL_SEC", "300")))
CITY_NEARBY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_NEARBY_CACHE_TTL_SEC", "480")))
CITY_MARKET_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_MARKET_CACHE_TTL_SEC", "900")))
CITY_HISTORY_PREVIEW_CACHE_TTL_SEC = max(
60,
int(os.getenv("POLYWEATHER_CITY_HISTORY_PREVIEW_CACHE_TTL_SEC", "1800")),
)
CACHE_REFRESH_LOCK_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CACHE_REFRESH_LOCK_TTL_SEC", "120")))
def _city_cache_is_fresh(entry: Optional[dict], ttl_sec: int) -> bool:
if not isinstance(entry, dict):
return False
updated_at_ts = float(entry.get("updated_at_ts") or 0.0)
if updated_at_ts <= 0:
return False
return (time.time() - updated_at_ts) < float(ttl_sec)
def _refresh_city_summary_cache(city: str, force_refresh: bool = False) -> dict:
data = _analyze_summary(city, force_refresh=force_refresh)
payload = _build_city_summary_payload(data)
_CACHE_DB.set_city_cache(
"summary",
city,
payload,
version="v1",
source_fingerprint=f"{city}:summary",
)
return payload
def _refresh_city_panel_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=False, detail_mode="panel")
_CACHE_DB.set_city_cache(
"panel",
city,
payload,
version="v1",
source_fingerprint=f"{city}:panel",
)
return payload
def _refresh_city_nearby_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=False, detail_mode="nearby")
_CACHE_DB.set_city_cache(
"nearby",
city,
payload,
version="v1",
source_fingerprint=f"{city}:nearby",
)
return payload
def _refresh_city_market_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=False, detail_mode="market")
_CACHE_DB.set_city_cache(
"market",
city,
payload,
version="v1",
source_fingerprint=f"{city}:market",
)
return payload
def _build_city_history_payload(city: str, include_records: bool = False) -> dict:
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
truth_rows = _truth_record_repo.load_city(city)
feature_rows = _training_feature_repo.load_city(city)
if not truth_rows and not feature_rows:
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
history_file = os.path.join(project_root, "data", "daily_records.json")
data = load_history(history_file)
city_data = data.get(city, {}) if isinstance(data.get(city, {}), dict) else {}
else:
all_dates = sorted(set(truth_rows.keys()) | set(feature_rows.keys()))
city_data = {}
for day in all_dates:
record: dict[str, object] = {}
truth = truth_rows.get(day) or {}
features = feature_rows.get(day) or {}
if truth.get("actual_high") is not None:
record["actual_high"] = truth.get("actual_high")
record["settlement_source"] = truth.get("settlement_source")
record["settlement_station_code"] = truth.get("settlement_station_code")
record["settlement_station_label"] = truth.get("settlement_station_label")
record["truth_version"] = truth.get("truth_version")
record["updated_by"] = truth.get("updated_by")
record["truth_updated_at"] = truth.get("truth_updated_at")
if isinstance(features, dict):
if features.get("deb_prediction") is not None:
record["deb_prediction"] = features.get("deb_prediction")
if features.get("mu") is not None:
record["mu"] = features.get("mu")
if isinstance(features.get("forecasts"), dict):
record["forecasts"] = features.get("forecasts")
city_data[day] = record
if not city_data:
return {
"history": [],
"mode": "full" if include_records else "preview",
"has_more": False,
"full_count": 0,
"preview_count": 0,
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
}
all_days = sorted(city_data.keys())
selected_days = all_days if include_records else all_days[-HISTORY_PREVIEW_DAY_LIMIT:]
out = []
for day in selected_days:
rec = city_data.get(day, {})
if not isinstance(rec, dict):
rec = {}
act = rec.get("actual_high")
deb = rec.get("deb_prediction")
mu = rec.get("mu")
snapshots = load_snapshot_rows_for_day(city, day)
peak_ref = _build_peak_minus_12h_reference(
actual_high=act,
snapshots=snapshots,
)
forecasts_raw = rec.get("forecasts", {}) or {}
forecasts = {}
if isinstance(forecasts_raw, dict):
for model_name, model_value in forecasts_raw.items():
if _is_excluded_model_name(str(model_name)):
continue
fv = _sf(model_value)
forecasts[str(model_name)] = fv if fv is not None else None
forecasts = _merge_missing_history_forecasts_from_snapshots(
forecasts,
snapshots,
)
mgm = forecasts.get("MGM")
out.append(
{
"date": day,
"actual": float(act) if act is not None else None,
"deb": float(deb) if deb is not None else None,
"mu": float(mu) if mu is not None else None,
"mgm": float(mgm) if mgm is not None else None,
"forecasts": forecasts,
"settlement_source": rec.get("settlement_source"),
"settlement_station_code": rec.get("settlement_station_code"),
"settlement_station_label": rec.get("settlement_station_label"),
"truth_version": rec.get("truth_version"),
"updated_by": rec.get("updated_by"),
"truth_updated_at": rec.get("truth_updated_at"),
"actual_peak_time": peak_ref.get("actual_peak_time"),
"deb_at_peak_minus_12h": peak_ref.get("deb_at_peak_minus_12h"),
"deb_at_peak_minus_12h_time": peak_ref.get("deb_at_peak_minus_12h_time"),
"deb_at_peak_minus_12h_error": peak_ref.get("deb_at_peak_minus_12h_error"),
}
)
return {
"history": out,
"mode": "full" if include_records else "preview",
"has_more": len(all_days) > len(selected_days),
"full_count": len(all_days),
"preview_count": len(out),
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
}
def _refresh_city_history_preview_cache(city: str) -> dict:
payload = _build_city_history_payload(city, include_records=False)
_CACHE_DB.set_city_cache(
"history_preview",
city,
payload,
version="v1",
source_fingerprint=f"{city}:history_preview",
)
return payload
def _schedule_cache_refresh(
background_tasks: BackgroundTasks,
*,
kind: str,
city: str,
force_refresh: bool = False,
) -> bool:
normalized_kind = str(kind or "").strip().lower()
normalized_city = str(city or "").strip().lower()
if normalized_kind not in {"summary", "panel", "nearby", "market", "history_preview"} or not normalized_city:
return False
cache_key = f"city:{normalized_kind}:{normalized_city}"
owner = _CACHE_DB.acquire_cache_refresh_lock(
cache_key,
ttl_sec=CACHE_REFRESH_LOCK_TTL_SEC,
)
if not owner:
return False
def _runner() -> None:
try:
if normalized_kind == "summary":
_refresh_city_summary_cache(normalized_city, force_refresh=force_refresh)
elif normalized_kind == "panel":
_refresh_city_panel_cache(normalized_city, force_refresh=force_refresh)
elif normalized_kind == "nearby":
_refresh_city_nearby_cache(normalized_city, force_refresh=force_refresh)
elif normalized_kind == "history_preview":
_refresh_city_history_preview_cache(normalized_city)
else:
_refresh_city_market_cache(normalized_city, force_refresh=force_refresh)
except Exception as exc:
logger.warning(
"cache refresh failed kind={} city={} force_refresh={}: {}",
normalized_kind,
normalized_city,
force_refresh,
exc,
)
finally:
_CACHE_DB.release_cache_refresh_lock(cache_key, owner)
background_tasks.add_task(_runner)
return True
def _parse_snapshot_dt(value: object) -> Optional[datetime]: def _parse_snapshot_dt(value: object) -> Optional[datetime]:
@@ -218,6 +492,33 @@ def _normalize_city_list(raw: Optional[str]) -> list[str]:
return out return out
def _select_priority_city_batches(client_timezone: Optional[str]) -> dict[str, object]:
tz = str(client_timezone or "").strip()
normalized = tz.lower()
if normalized.startswith("america/"):
primary = list(US_CORE_CITIES)
secondary = []
region = "america"
elif normalized.startswith("europe/"):
primary = list(EUROPE_CORE_CITIES)
secondary = list(ASIA_CORE_CITIES)
region = "europe"
elif normalized.startswith("asia/") or normalized.startswith("australia/") or normalized.startswith("pacific/"):
primary = list(ASIA_CORE_CITIES)
secondary = list(EUROPE_CORE_CITIES)
region = "asia"
else:
primary = list(ASIA_CORE_CITIES)
secondary = list(EUROPE_CORE_CITIES)
region = "default"
return {
"region": region,
"timezone": tz or None,
"primary": primary,
"secondary": secondary,
}
def _history_file_path() -> str: def _history_file_path() -> str:
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
return os.path.join(project_root, "data", "daily_records.json") return os.path.join(project_root, "data", "daily_records.json")
@@ -317,7 +618,7 @@ async def system_prewarm(
for city in selected: for city in selected:
city_started = time.perf_counter() city_started = time.perf_counter()
try: try:
data = _analyze(city, force_refresh=force_refresh) _refresh_city_summary_cache(city, force_refresh=force_refresh)
entry: dict[str, object] = { entry: dict[str, object] = {
"city": city, "city": city,
"summary": True, "summary": True,
@@ -325,20 +626,11 @@ async def system_prewarm(
} }
summary_ok += 1 summary_ok += 1
if include_detail: if include_detail:
_build_city_summary_payload(data) _refresh_city_panel_cache(city, force_refresh=force_refresh)
_build_city_detail_payload(
data,
target_date=str(data.get("local_date") or "").strip() or None,
)
entry["detail"] = True entry["detail"] = True
detail_ok += 1 detail_ok += 1
if include_market: if include_market:
_build_city_detail_payload( entry["market"] = False
data,
target_date=str(data.get("local_date") or "").strip() or None,
)
entry["market"] = True
market_ok += 1
warmed.append(entry) warmed.append(entry)
except Exception as exc: except Exception as exc:
failed.append( failed.append(
@@ -365,6 +657,7 @@ async def system_prewarm(
"warmed": warmed, "warmed": warmed,
"failed": failed, "failed": failed,
"summary_ok": summary_ok, "summary_ok": summary_ok,
"panel_ok": detail_ok,
"detail_ok": detail_ok, "detail_ok": detail_ok,
"market_ok": market_ok, "market_ok": market_ok,
"failed_count": len(failed), "failed_count": len(failed),
@@ -431,6 +724,7 @@ async def list_cities(request: Request):
@router.get("/api/city/{name}") @router.get("/api/city/{name}")
async def city_detail( async def city_detail(
request: Request, request: Request,
background_tasks: BackgroundTasks,
name: str, name: str,
force_refresh: bool = False, force_refresh: bool = False,
depth: str = "panel", depth: str = "panel",
@@ -446,128 +740,121 @@ async def city_detail(
detail_mode = "nearby" detail_mode = "nearby"
else: else:
detail_mode = "panel" detail_mode = "panel"
if detail_mode == "panel":
if force_refresh:
return await run_in_threadpool(_refresh_city_panel_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "panel", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_PANEL_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="panel", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_panel_cache, city, False)
if detail_mode == "nearby":
if force_refresh:
return await run_in_threadpool(_refresh_city_nearby_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "nearby", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_NEARBY_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="nearby", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_nearby_cache, city, False)
if detail_mode == "market":
if force_refresh:
return await run_in_threadpool(_refresh_city_market_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "market", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_MARKET_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="market", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_market_cache, city, False)
return await run_in_threadpool(_analyze, city, force_refresh, False, detail_mode) return await run_in_threadpool(_analyze, city, force_refresh, False, detail_mode)
@router.get("/api/history/{name}") @router.get("/api/history/{name}")
async def city_history( async def city_history(
request: Request, request: Request,
background_tasks: BackgroundTasks,
name: str, name: str,
include_records: bool = False, include_records: bool = False,
): ):
_assert_entitlement(request) _assert_entitlement(request)
city = _normalize_city_or_404(name) city = _normalize_city_or_404(name)
if include_records:
return await run_in_threadpool(_build_city_history_payload, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "history_preview", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_HISTORY_PREVIEW_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="history_preview", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_history_preview_cache, city)
def _build_history_payload():
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
truth_rows = _truth_record_repo.load_city(city)
feature_rows = _training_feature_repo.load_city(city)
if not truth_rows and not feature_rows: @router.get("/api/system/cache-status")
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) async def system_cache_status(request: Request, cities: Optional[str] = None):
history_file = os.path.join(project_root, "data", "daily_records.json") _assert_entitlement(request)
data = load_history(history_file) selected = _normalize_city_list(cities)
city_data = data.get(city, {}) if isinstance(data.get(city, {}), dict) else {} if not selected:
else: selected = list(DEFAULT_PREWARM_CITIES)
all_dates = sorted(set(truth_rows.keys()) | set(feature_rows.keys())) kinds = {
city_data = {} "summary": CITY_SUMMARY_CACHE_TTL_SEC,
for day in all_dates: "panel": CITY_PANEL_CACHE_TTL_SEC,
record: dict[str, object] = {} "nearby": CITY_NEARBY_CACHE_TTL_SEC,
truth = truth_rows.get(day) or {} "market": CITY_MARKET_CACHE_TTL_SEC,
features = feature_rows.get(day) or {} "history_preview": CITY_HISTORY_PREVIEW_CACHE_TTL_SEC,
if truth.get("actual_high") is not None: }
record["actual_high"] = truth.get("actual_high") items = []
record["settlement_source"] = truth.get("settlement_source") for city in selected:
record["settlement_station_code"] = truth.get("settlement_station_code") row = {"city": city}
record["settlement_station_label"] = truth.get("settlement_station_label") for kind, ttl_sec in kinds.items():
record["truth_version"] = truth.get("truth_version") entry = _CACHE_DB.get_city_cache(kind, city)
record["updated_by"] = truth.get("updated_by") row[kind] = {
record["truth_updated_at"] = truth.get("truth_updated_at") "exists": bool(entry),
if isinstance(features, dict): "fresh": _city_cache_is_fresh(entry, ttl_sec),
if features.get("deb_prediction") is not None: "updated_at": entry.get("updated_at") if entry else None,
record["deb_prediction"] = features.get("deb_prediction") "age_sec": round(max(0.0, time.time() - float(entry.get("updated_at_ts") or 0.0)), 1)
if features.get("mu") is not None: if entry
record["mu"] = features.get("mu") else None,
if isinstance(features.get("forecasts"), dict): "ttl_sec": ttl_sec,
record["forecasts"] = features.get("forecasts")
city_data[day] = record
if not city_data:
return {
"history": [],
"mode": "full" if include_records else "preview",
"has_more": False,
"full_count": 0,
"preview_count": 0,
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
} }
items.append(row)
return {"cities": items}
all_days = sorted(city_data.keys())
selected_days = (
all_days
if include_records
else all_days[-HISTORY_PREVIEW_DAY_LIMIT:]
)
out = []
for day in selected_days:
rec = city_data.get(day, {})
if not isinstance(rec, dict):
rec = {}
act = rec.get("actual_high") @router.post("/api/system/priority-warm")
deb = rec.get("deb_prediction") async def system_priority_warm(
mu = rec.get("mu") request: Request,
snapshots = load_snapshot_rows_for_day(city, day) background_tasks: BackgroundTasks,
peak_ref = _build_peak_minus_12h_reference( timezone: Optional[str] = None,
actual_high=act, ):
snapshots=snapshots, _assert_entitlement(request)
) batches = _select_priority_city_batches(timezone)
forecasts_raw = rec.get("forecasts", {}) or {} primary = list(batches.get("primary") or [])
forecasts = {} secondary = list(batches.get("secondary") or [])
if isinstance(forecasts_raw, dict):
for model_name, model_value in forecasts_raw.items():
if _is_excluded_model_name(str(model_name)):
continue
fv = _sf(model_value)
forecasts[str(model_name)] = fv if fv is not None else None
forecasts = _merge_missing_history_forecasts_from_snapshots(
forecasts,
snapshots,
)
mgm = forecasts.get("MGM")
out.append(
{
"date": day,
"actual": float(act) if act is not None else None,
"deb": float(deb) if deb is not None else None,
"mu": float(mu) if mu is not None else None,
"mgm": float(mgm) if mgm is not None else None,
"forecasts": forecasts,
"settlement_source": rec.get("settlement_source"),
"settlement_station_code": rec.get("settlement_station_code"),
"settlement_station_label": rec.get("settlement_station_label"),
"truth_version": rec.get("truth_version"),
"updated_by": rec.get("updated_by"),
"truth_updated_at": rec.get("truth_updated_at"),
"actual_peak_time": peak_ref.get("actual_peak_time"),
"deb_at_peak_minus_12h": peak_ref.get("deb_at_peak_minus_12h"),
"deb_at_peak_minus_12h_time": peak_ref.get("deb_at_peak_minus_12h_time"),
"deb_at_peak_minus_12h_error": peak_ref.get("deb_at_peak_minus_12h_error"),
}
)
return { def _runner() -> None:
"history": out, for city in primary:
"mode": "full" if include_records else "preview", try:
"has_more": len(all_days) > len(selected_days), _refresh_city_summary_cache(city, force_refresh=False)
"full_count": len(all_days), _refresh_city_panel_cache(city, force_refresh=False)
"preview_count": len(out), _refresh_city_nearby_cache(city, force_refresh=False)
"settlement_source": source, _refresh_city_market_cache(city, force_refresh=False)
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()), except Exception as exc:
} logger.warning("priority warm primary failed city={} timezone={}: {}", city, timezone, exc)
for city in secondary:
try:
_refresh_city_summary_cache(city, force_refresh=False)
_refresh_city_panel_cache(city, force_refresh=False)
except Exception as exc:
logger.warning("priority warm secondary failed city={} timezone={}: {}", city, timezone, exc)
return await run_in_threadpool(_build_history_payload) background_tasks.add_task(_runner)
return {
"ok": True,
"region": batches.get("region"),
"timezone": batches.get("timezone"),
"primary": primary,
"secondary": secondary,
}
@router.get("/api/auth/me") @router.get("/api/auth/me")
@@ -1063,10 +1350,21 @@ async def payment_reconcile_latest(request: Request):
@router.get("/api/city/{name}/summary") @router.get("/api/city/{name}/summary")
async def city_summary(request: Request, name: str, force_refresh: bool = False): async def city_summary(
request: Request,
background_tasks: BackgroundTasks,
name: str,
force_refresh: bool = False,
):
city = _normalize_city_or_404(name) city = _normalize_city_or_404(name)
data = await run_in_threadpool(_analyze_summary, city, force_refresh) if force_refresh:
return await run_in_threadpool(_build_city_summary_payload, data) return await run_in_threadpool(_refresh_city_summary_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "summary", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_SUMMARY_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="summary", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_summary_cache, city, False)
@router.get("/api/city/{name}/detail") @router.get("/api/city/{name}/detail")
@@ -1091,6 +1389,7 @@ async def city_detail_aggregate(
@router.get("/api/city/{name}/market-scan") @router.get("/api/city/{name}/market-scan")
async def city_market_scan( async def city_market_scan(
request: Request, request: Request,
background_tasks: BackgroundTasks,
name: str, name: str,
force_refresh: bool = False, force_refresh: bool = False,
market_slug: Optional[str] = None, market_slug: Optional[str] = None,
@@ -1098,7 +1397,16 @@ async def city_market_scan(
): ):
_assert_entitlement(request) _assert_entitlement(request)
city = _normalize_city_or_404(name) city = _normalize_city_or_404(name)
data = await run_in_threadpool(_analyze, city, force_refresh, False, "market") if force_refresh:
data = await run_in_threadpool(_refresh_city_market_cache, city, True)
else:
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "market", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_MARKET_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="market", city=city)
data = cached_entry.get("payload") or {}
else:
data = await run_in_threadpool(_refresh_city_market_cache, city, False)
return await run_in_threadpool( return await run_in_threadpool(
_build_city_market_scan_payload, _build_city_market_scan_payload,
data, data,