Add cached dashboard prewarm hints for priority cities

This commit is contained in:
2569718930@qq.com
2026-04-16 15:05:14 +08:00
parent b635070106
commit c0e5307893
5 changed files with 672 additions and 123 deletions
+19
View File
@@ -618,6 +618,25 @@ export function DashboardStoreProvider({
void loadCities();
}, []);
useEffect(() => {
if (!cities.length) return;
if (typeof window === "undefined") return;
const schedule = () => dashboardClient.sendPriorityWarmHint();
const idleCallback = (window as Window & {
requestIdleCallback?: (callback: () => void, options?: { timeout: number }) => number;
}).requestIdleCallback;
if (typeof idleCallback === "function") {
const id = idleCallback(schedule, { timeout: 3000 });
return () => {
if (typeof window.cancelIdleCallback === "function") {
window.cancelIdleCallback(id);
}
};
}
const timer = window.setTimeout(schedule, 2000);
return () => window.clearTimeout(timer);
}, [cities.length]);
useEffect(() => {
void refreshProAccess();
}, []);
+19
View File
@@ -12,6 +12,7 @@ const CACHE_TTL_MS = 5 * 60 * 1000;
const pendingCityDetailRequests = new Map<string, Promise<CityDetail>>();
const pendingHistoryRequests = new Map<string, Promise<HistoryPayload>>();
const pendingCitySummaryRequests = new Map<string, Promise<CitySummary>>();
const PRIORITY_WARM_SESSION_KEY = "polyWeather_priority_warm_v1";
type CityCacheMeta = {
cachedAt: number;
@@ -142,6 +143,24 @@ export const dashboardClient = {
return data.cities || [];
},
sendPriorityWarmHint(timezone?: string | null) {
if (!isClient()) return;
const tz = String(
timezone || Intl.DateTimeFormat().resolvedOptions().timeZone || "",
).trim();
if (!tz) return;
const cacheKey = `${PRIORITY_WARM_SESSION_KEY}:${tz}`;
if (window.sessionStorage.getItem(cacheKey)) return;
window.sessionStorage.setItem(cacheKey, "1");
const params = new URLSearchParams({ timezone: tz });
void fetch(`/api/system/priority-warm?${params.toString()}`, {
method: "POST",
headers: { Accept: "application/json" },
cache: "no-store",
keepalive: true,
}).catch(() => {});
},
async getCitySummary(cityName: string, options?: { force?: boolean }) {
const force = options?.force ?? false;
const requestKey = `${cityName}::${force ? "force" : "cached"}`;
+203
View File
@@ -182,6 +182,64 @@ class DBManager:
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_summary_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_panel_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_nearby_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_market_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS city_history_preview_cache (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
version TEXT,
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS cache_refresh_locks (
cache_key TEXT PRIMARY KEY,
locked_until_ts REAL NOT NULL,
owner TEXT NOT NULL,
updated_at TEXT NOT NULL
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS payment_audit_events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -247,6 +305,151 @@ class DBManager:
conn.commit()
logger.info(f"Database initialized successfully path={self.db_path}")
def _cache_table_name(self, kind: str) -> Optional[str]:
normalized = str(kind or "").strip().lower()
if normalized == "summary":
return "city_summary_cache"
if normalized == "panel":
return "city_panel_cache"
if normalized == "nearby":
return "city_nearby_cache"
if normalized == "market":
return "city_market_cache"
if normalized == "history_preview":
return "city_history_preview_cache"
return None
def get_city_cache(self, kind: str, city: str) -> Optional[Dict[str, Any]]:
table = self._cache_table_name(kind)
normalized_city = str(city or "").strip().lower()
if not table or not normalized_city:
return None
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
row = conn.execute(
f"""
SELECT city, payload_json, updated_at, updated_at_ts, version, source_fingerprint
FROM {table}
WHERE city = ?
LIMIT 1
""",
(normalized_city,),
).fetchone()
if not row:
return None
try:
payload = json.loads(str(row["payload_json"] or "{}"))
except Exception:
return None
if not isinstance(payload, dict):
return None
return {
"city": str(row["city"] or normalized_city),
"payload": payload,
"updated_at": str(row["updated_at"] or ""),
"updated_at_ts": float(row["updated_at_ts"] or 0.0),
"version": str(row["version"] or ""),
"source_fingerprint": str(row["source_fingerprint"] or ""),
}
def set_city_cache(
self,
kind: str,
city: str,
payload: Dict[str, Any],
*,
version: str = "v1",
source_fingerprint: Optional[str] = None,
) -> None:
table = self._cache_table_name(kind)
normalized_city = str(city or "").strip().lower()
if not table or not normalized_city or not isinstance(payload, dict):
return
now = datetime.now().isoformat()
now_ts = datetime.now().timestamp()
with self._get_connection() as conn:
conn.execute(
f"""
INSERT INTO {table} (
city,
payload_json,
updated_at,
updated_at_ts,
version,
source_fingerprint
)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(city) DO UPDATE SET
payload_json = excluded.payload_json,
updated_at = excluded.updated_at,
updated_at_ts = excluded.updated_at_ts,
version = excluded.version,
source_fingerprint = excluded.source_fingerprint
""",
(
normalized_city,
json.dumps(payload, ensure_ascii=False),
now,
now_ts,
str(version or "v1"),
str(source_fingerprint or ""),
),
)
conn.commit()
def acquire_cache_refresh_lock(
self,
cache_key: str,
*,
ttl_sec: int = 120,
owner: Optional[str] = None,
) -> Optional[str]:
normalized_key = str(cache_key or "").strip().lower()
if not normalized_key:
return None
lock_owner = str(owner or "").strip() or hashlib.sha1(
f"{normalized_key}:{datetime.now().timestamp()}".encode("utf-8")
).hexdigest()[:12]
now_ts = datetime.now().timestamp()
locked_until_ts = now_ts + max(15, int(ttl_sec or 120))
updated_at = datetime.now().isoformat()
with self._get_connection() as conn:
cursor = conn.execute(
"""
INSERT INTO cache_refresh_locks (cache_key, locked_until_ts, owner, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(cache_key) DO UPDATE SET
locked_until_ts = excluded.locked_until_ts,
owner = excluded.owner,
updated_at = excluded.updated_at
WHERE cache_refresh_locks.locked_until_ts < ?
""",
(
normalized_key,
locked_until_ts,
lock_owner,
updated_at,
now_ts,
),
)
conn.commit()
return lock_owner if int(cursor.rowcount or 0) > 0 else None
def release_cache_refresh_lock(self, cache_key: str, owner: str) -> None:
normalized_key = str(cache_key or "").strip().lower()
normalized_owner = str(owner or "").strip()
if not normalized_key or not normalized_owner:
return
with self._get_connection() as conn:
conn.execute(
"""
DELETE FROM cache_refresh_locks
WHERE cache_key = ? AND owner = ?
""",
(normalized_key, normalized_owner),
)
conn.commit()
def get_payment_runtime_state(self, state_key: str) -> Optional[Dict[str, Any]]:
key = str(state_key or "").strip()
if not key:
+2 -2
View File
@@ -208,7 +208,7 @@ def get_prewarm_runtime_summary() -> Dict[str, Any]:
"interval_sec": interval_sec,
"jitter_sec": jitter_sec,
"include_detail": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_DETAIL", True),
"include_market": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_MARKET", True),
"include_market": _truthy_env("POLYWEATHER_PREWARM_INCLUDE_MARKET", False),
"force_refresh": _truthy_env("POLYWEATHER_PREWARM_FORCE_REFRESH", False),
"thread_alive": bool(_WORKER_THREAD and _WORKER_THREAD.is_alive()) or shared_alive,
"heartbeat_age_sec": None if heartbeat_age_sec is None else round(heartbeat_age_sec, 2),
@@ -380,7 +380,7 @@ def start_prewarm_worker_thread() -> Optional[threading.Thread]:
jitter_sec = int(os.getenv("POLYWEATHER_PREWARM_JITTER_SEC", "20"))
force_refresh = str(os.getenv("POLYWEATHER_PREWARM_FORCE_REFRESH") or "").strip().lower() in {"1", "true", "yes", "on"}
include_detail = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_DETAIL", "true")).strip().lower() in {"1", "true", "yes", "on"}
include_market = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_MARKET", "true")).strip().lower() in {"1", "true", "yes", "on"}
include_market = str(os.getenv("POLYWEATHER_PREWARM_INCLUDE_MARKET", "false")).strip().lower() in {"1", "true", "yes", "on"}
thread = threading.Thread(
target=run_worker_loop,
+429 -121
View File
@@ -6,12 +6,13 @@ from datetime import datetime, timedelta
from typing import Optional
from fastapi.concurrency import run_in_threadpool
from fastapi import APIRouter, HTTPException, Request
from fastapi import APIRouter, BackgroundTasks, HTTPException, Request
from fastapi.responses import PlainTextResponse
from loguru import logger
from src.analysis.deb_algorithm import load_history
from src.analysis.probability_snapshot_archive import load_snapshot_rows_for_day
from src.database.db_manager import DBManager
from src.database.runtime_state import TrainingFeatureRecordRepository, TruthRecordRepository
from src.analysis.settlement_rounding import apply_city_settlement
from src.data_collection.country_networks import get_country_network_provider
@@ -55,6 +56,7 @@ from web.core import (
)
router = APIRouter()
_CACHE_DB = DBManager()
_DEB_RECENT_LOOKBACK = 7
_DEB_RECENT_MIN_SAMPLES = 3
@@ -91,6 +93,278 @@ DEFAULT_PREWARM_CITIES = [
"madrid",
]
HISTORY_PREVIEW_DAY_LIMIT = 21
ASIA_CORE_CITIES = [
"hong kong",
"taipei",
"tokyo",
"seoul",
"busan",
"shanghai",
"beijing",
"guangzhou",
"shenzhen",
"chongqing",
"chengdu",
"singapore",
"kuala lumpur",
"jakarta",
]
EUROPE_CORE_CITIES = [
"istanbul",
"ankara",
"moscow",
"tel aviv",
"london",
"paris",
"madrid",
"milan",
"warsaw",
"amsterdam",
"helsinki",
]
US_CORE_CITIES = [
"new york",
"los angeles",
"san francisco",
"austin",
"houston",
"chicago",
"dallas",
"miami",
"atlanta",
"seattle",
]
CITY_SUMMARY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_SUMMARY_CACHE_TTL_SEC", "180")))
CITY_PANEL_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_PANEL_CACHE_TTL_SEC", "300")))
CITY_NEARBY_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_NEARBY_CACHE_TTL_SEC", "480")))
CITY_MARKET_CACHE_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CITY_MARKET_CACHE_TTL_SEC", "900")))
CITY_HISTORY_PREVIEW_CACHE_TTL_SEC = max(
60,
int(os.getenv("POLYWEATHER_CITY_HISTORY_PREVIEW_CACHE_TTL_SEC", "1800")),
)
CACHE_REFRESH_LOCK_TTL_SEC = max(30, int(os.getenv("POLYWEATHER_CACHE_REFRESH_LOCK_TTL_SEC", "120")))
def _city_cache_is_fresh(entry: Optional[dict], ttl_sec: int) -> bool:
if not isinstance(entry, dict):
return False
updated_at_ts = float(entry.get("updated_at_ts") or 0.0)
if updated_at_ts <= 0:
return False
return (time.time() - updated_at_ts) < float(ttl_sec)
def _refresh_city_summary_cache(city: str, force_refresh: bool = False) -> dict:
data = _analyze_summary(city, force_refresh=force_refresh)
payload = _build_city_summary_payload(data)
_CACHE_DB.set_city_cache(
"summary",
city,
payload,
version="v1",
source_fingerprint=f"{city}:summary",
)
return payload
def _refresh_city_panel_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=False, detail_mode="panel")
_CACHE_DB.set_city_cache(
"panel",
city,
payload,
version="v1",
source_fingerprint=f"{city}:panel",
)
return payload
def _refresh_city_nearby_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=False, detail_mode="nearby")
_CACHE_DB.set_city_cache(
"nearby",
city,
payload,
version="v1",
source_fingerprint=f"{city}:nearby",
)
return payload
def _refresh_city_market_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, include_llm_commentary=False, detail_mode="market")
_CACHE_DB.set_city_cache(
"market",
city,
payload,
version="v1",
source_fingerprint=f"{city}:market",
)
return payload
def _build_city_history_payload(city: str, include_records: bool = False) -> dict:
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
truth_rows = _truth_record_repo.load_city(city)
feature_rows = _training_feature_repo.load_city(city)
if not truth_rows and not feature_rows:
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
history_file = os.path.join(project_root, "data", "daily_records.json")
data = load_history(history_file)
city_data = data.get(city, {}) if isinstance(data.get(city, {}), dict) else {}
else:
all_dates = sorted(set(truth_rows.keys()) | set(feature_rows.keys()))
city_data = {}
for day in all_dates:
record: dict[str, object] = {}
truth = truth_rows.get(day) or {}
features = feature_rows.get(day) or {}
if truth.get("actual_high") is not None:
record["actual_high"] = truth.get("actual_high")
record["settlement_source"] = truth.get("settlement_source")
record["settlement_station_code"] = truth.get("settlement_station_code")
record["settlement_station_label"] = truth.get("settlement_station_label")
record["truth_version"] = truth.get("truth_version")
record["updated_by"] = truth.get("updated_by")
record["truth_updated_at"] = truth.get("truth_updated_at")
if isinstance(features, dict):
if features.get("deb_prediction") is not None:
record["deb_prediction"] = features.get("deb_prediction")
if features.get("mu") is not None:
record["mu"] = features.get("mu")
if isinstance(features.get("forecasts"), dict):
record["forecasts"] = features.get("forecasts")
city_data[day] = record
if not city_data:
return {
"history": [],
"mode": "full" if include_records else "preview",
"has_more": False,
"full_count": 0,
"preview_count": 0,
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
}
all_days = sorted(city_data.keys())
selected_days = all_days if include_records else all_days[-HISTORY_PREVIEW_DAY_LIMIT:]
out = []
for day in selected_days:
rec = city_data.get(day, {})
if not isinstance(rec, dict):
rec = {}
act = rec.get("actual_high")
deb = rec.get("deb_prediction")
mu = rec.get("mu")
snapshots = load_snapshot_rows_for_day(city, day)
peak_ref = _build_peak_minus_12h_reference(
actual_high=act,
snapshots=snapshots,
)
forecasts_raw = rec.get("forecasts", {}) or {}
forecasts = {}
if isinstance(forecasts_raw, dict):
for model_name, model_value in forecasts_raw.items():
if _is_excluded_model_name(str(model_name)):
continue
fv = _sf(model_value)
forecasts[str(model_name)] = fv if fv is not None else None
forecasts = _merge_missing_history_forecasts_from_snapshots(
forecasts,
snapshots,
)
mgm = forecasts.get("MGM")
out.append(
{
"date": day,
"actual": float(act) if act is not None else None,
"deb": float(deb) if deb is not None else None,
"mu": float(mu) if mu is not None else None,
"mgm": float(mgm) if mgm is not None else None,
"forecasts": forecasts,
"settlement_source": rec.get("settlement_source"),
"settlement_station_code": rec.get("settlement_station_code"),
"settlement_station_label": rec.get("settlement_station_label"),
"truth_version": rec.get("truth_version"),
"updated_by": rec.get("updated_by"),
"truth_updated_at": rec.get("truth_updated_at"),
"actual_peak_time": peak_ref.get("actual_peak_time"),
"deb_at_peak_minus_12h": peak_ref.get("deb_at_peak_minus_12h"),
"deb_at_peak_minus_12h_time": peak_ref.get("deb_at_peak_minus_12h_time"),
"deb_at_peak_minus_12h_error": peak_ref.get("deb_at_peak_minus_12h_error"),
}
)
return {
"history": out,
"mode": "full" if include_records else "preview",
"has_more": len(all_days) > len(selected_days),
"full_count": len(all_days),
"preview_count": len(out),
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
}
def _refresh_city_history_preview_cache(city: str) -> dict:
payload = _build_city_history_payload(city, include_records=False)
_CACHE_DB.set_city_cache(
"history_preview",
city,
payload,
version="v1",
source_fingerprint=f"{city}:history_preview",
)
return payload
def _schedule_cache_refresh(
background_tasks: BackgroundTasks,
*,
kind: str,
city: str,
force_refresh: bool = False,
) -> bool:
normalized_kind = str(kind or "").strip().lower()
normalized_city = str(city or "").strip().lower()
if normalized_kind not in {"summary", "panel", "nearby", "market", "history_preview"} or not normalized_city:
return False
cache_key = f"city:{normalized_kind}:{normalized_city}"
owner = _CACHE_DB.acquire_cache_refresh_lock(
cache_key,
ttl_sec=CACHE_REFRESH_LOCK_TTL_SEC,
)
if not owner:
return False
def _runner() -> None:
try:
if normalized_kind == "summary":
_refresh_city_summary_cache(normalized_city, force_refresh=force_refresh)
elif normalized_kind == "panel":
_refresh_city_panel_cache(normalized_city, force_refresh=force_refresh)
elif normalized_kind == "nearby":
_refresh_city_nearby_cache(normalized_city, force_refresh=force_refresh)
elif normalized_kind == "history_preview":
_refresh_city_history_preview_cache(normalized_city)
else:
_refresh_city_market_cache(normalized_city, force_refresh=force_refresh)
except Exception as exc:
logger.warning(
"cache refresh failed kind={} city={} force_refresh={}: {}",
normalized_kind,
normalized_city,
force_refresh,
exc,
)
finally:
_CACHE_DB.release_cache_refresh_lock(cache_key, owner)
background_tasks.add_task(_runner)
return True
def _parse_snapshot_dt(value: object) -> Optional[datetime]:
@@ -218,6 +492,33 @@ def _normalize_city_list(raw: Optional[str]) -> list[str]:
return out
def _select_priority_city_batches(client_timezone: Optional[str]) -> dict[str, object]:
tz = str(client_timezone or "").strip()
normalized = tz.lower()
if normalized.startswith("america/"):
primary = list(US_CORE_CITIES)
secondary = []
region = "america"
elif normalized.startswith("europe/"):
primary = list(EUROPE_CORE_CITIES)
secondary = list(ASIA_CORE_CITIES)
region = "europe"
elif normalized.startswith("asia/") or normalized.startswith("australia/") or normalized.startswith("pacific/"):
primary = list(ASIA_CORE_CITIES)
secondary = list(EUROPE_CORE_CITIES)
region = "asia"
else:
primary = list(ASIA_CORE_CITIES)
secondary = list(EUROPE_CORE_CITIES)
region = "default"
return {
"region": region,
"timezone": tz or None,
"primary": primary,
"secondary": secondary,
}
def _history_file_path() -> str:
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
return os.path.join(project_root, "data", "daily_records.json")
@@ -317,7 +618,7 @@ async def system_prewarm(
for city in selected:
city_started = time.perf_counter()
try:
data = _analyze(city, force_refresh=force_refresh)
_refresh_city_summary_cache(city, force_refresh=force_refresh)
entry: dict[str, object] = {
"city": city,
"summary": True,
@@ -325,20 +626,11 @@ async def system_prewarm(
}
summary_ok += 1
if include_detail:
_build_city_summary_payload(data)
_build_city_detail_payload(
data,
target_date=str(data.get("local_date") or "").strip() or None,
)
_refresh_city_panel_cache(city, force_refresh=force_refresh)
entry["detail"] = True
detail_ok += 1
if include_market:
_build_city_detail_payload(
data,
target_date=str(data.get("local_date") or "").strip() or None,
)
entry["market"] = True
market_ok += 1
entry["market"] = False
warmed.append(entry)
except Exception as exc:
failed.append(
@@ -365,6 +657,7 @@ async def system_prewarm(
"warmed": warmed,
"failed": failed,
"summary_ok": summary_ok,
"panel_ok": detail_ok,
"detail_ok": detail_ok,
"market_ok": market_ok,
"failed_count": len(failed),
@@ -431,6 +724,7 @@ async def list_cities(request: Request):
@router.get("/api/city/{name}")
async def city_detail(
request: Request,
background_tasks: BackgroundTasks,
name: str,
force_refresh: bool = False,
depth: str = "panel",
@@ -446,128 +740,121 @@ async def city_detail(
detail_mode = "nearby"
else:
detail_mode = "panel"
if detail_mode == "panel":
if force_refresh:
return await run_in_threadpool(_refresh_city_panel_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "panel", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_PANEL_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="panel", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_panel_cache, city, False)
if detail_mode == "nearby":
if force_refresh:
return await run_in_threadpool(_refresh_city_nearby_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "nearby", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_NEARBY_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="nearby", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_nearby_cache, city, False)
if detail_mode == "market":
if force_refresh:
return await run_in_threadpool(_refresh_city_market_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "market", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_MARKET_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="market", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_market_cache, city, False)
return await run_in_threadpool(_analyze, city, force_refresh, False, detail_mode)
@router.get("/api/history/{name}")
async def city_history(
request: Request,
background_tasks: BackgroundTasks,
name: str,
include_records: bool = False,
):
_assert_entitlement(request)
city = _normalize_city_or_404(name)
if include_records:
return await run_in_threadpool(_build_city_history_payload, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "history_preview", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_HISTORY_PREVIEW_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="history_preview", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_history_preview_cache, city)
def _build_history_payload():
source = str(CITIES.get(city, {}).get("settlement_source") or "metar").strip().lower()
truth_rows = _truth_record_repo.load_city(city)
feature_rows = _training_feature_repo.load_city(city)
if not truth_rows and not feature_rows:
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
history_file = os.path.join(project_root, "data", "daily_records.json")
data = load_history(history_file)
city_data = data.get(city, {}) if isinstance(data.get(city, {}), dict) else {}
else:
all_dates = sorted(set(truth_rows.keys()) | set(feature_rows.keys()))
city_data = {}
for day in all_dates:
record: dict[str, object] = {}
truth = truth_rows.get(day) or {}
features = feature_rows.get(day) or {}
if truth.get("actual_high") is not None:
record["actual_high"] = truth.get("actual_high")
record["settlement_source"] = truth.get("settlement_source")
record["settlement_station_code"] = truth.get("settlement_station_code")
record["settlement_station_label"] = truth.get("settlement_station_label")
record["truth_version"] = truth.get("truth_version")
record["updated_by"] = truth.get("updated_by")
record["truth_updated_at"] = truth.get("truth_updated_at")
if isinstance(features, dict):
if features.get("deb_prediction") is not None:
record["deb_prediction"] = features.get("deb_prediction")
if features.get("mu") is not None:
record["mu"] = features.get("mu")
if isinstance(features.get("forecasts"), dict):
record["forecasts"] = features.get("forecasts")
city_data[day] = record
if not city_data:
return {
"history": [],
"mode": "full" if include_records else "preview",
"has_more": False,
"full_count": 0,
"preview_count": 0,
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
@router.get("/api/system/cache-status")
async def system_cache_status(request: Request, cities: Optional[str] = None):
_assert_entitlement(request)
selected = _normalize_city_list(cities)
if not selected:
selected = list(DEFAULT_PREWARM_CITIES)
kinds = {
"summary": CITY_SUMMARY_CACHE_TTL_SEC,
"panel": CITY_PANEL_CACHE_TTL_SEC,
"nearby": CITY_NEARBY_CACHE_TTL_SEC,
"market": CITY_MARKET_CACHE_TTL_SEC,
"history_preview": CITY_HISTORY_PREVIEW_CACHE_TTL_SEC,
}
items = []
for city in selected:
row = {"city": city}
for kind, ttl_sec in kinds.items():
entry = _CACHE_DB.get_city_cache(kind, city)
row[kind] = {
"exists": bool(entry),
"fresh": _city_cache_is_fresh(entry, ttl_sec),
"updated_at": entry.get("updated_at") if entry else None,
"age_sec": round(max(0.0, time.time() - float(entry.get("updated_at_ts") or 0.0)), 1)
if entry
else None,
"ttl_sec": ttl_sec,
}
items.append(row)
return {"cities": items}
all_days = sorted(city_data.keys())
selected_days = (
all_days
if include_records
else all_days[-HISTORY_PREVIEW_DAY_LIMIT:]
)
out = []
for day in selected_days:
rec = city_data.get(day, {})
if not isinstance(rec, dict):
rec = {}
act = rec.get("actual_high")
deb = rec.get("deb_prediction")
mu = rec.get("mu")
snapshots = load_snapshot_rows_for_day(city, day)
peak_ref = _build_peak_minus_12h_reference(
actual_high=act,
snapshots=snapshots,
)
forecasts_raw = rec.get("forecasts", {}) or {}
forecasts = {}
if isinstance(forecasts_raw, dict):
for model_name, model_value in forecasts_raw.items():
if _is_excluded_model_name(str(model_name)):
continue
fv = _sf(model_value)
forecasts[str(model_name)] = fv if fv is not None else None
forecasts = _merge_missing_history_forecasts_from_snapshots(
forecasts,
snapshots,
)
mgm = forecasts.get("MGM")
out.append(
{
"date": day,
"actual": float(act) if act is not None else None,
"deb": float(deb) if deb is not None else None,
"mu": float(mu) if mu is not None else None,
"mgm": float(mgm) if mgm is not None else None,
"forecasts": forecasts,
"settlement_source": rec.get("settlement_source"),
"settlement_station_code": rec.get("settlement_station_code"),
"settlement_station_label": rec.get("settlement_station_label"),
"truth_version": rec.get("truth_version"),
"updated_by": rec.get("updated_by"),
"truth_updated_at": rec.get("truth_updated_at"),
"actual_peak_time": peak_ref.get("actual_peak_time"),
"deb_at_peak_minus_12h": peak_ref.get("deb_at_peak_minus_12h"),
"deb_at_peak_minus_12h_time": peak_ref.get("deb_at_peak_minus_12h_time"),
"deb_at_peak_minus_12h_error": peak_ref.get("deb_at_peak_minus_12h_error"),
}
)
@router.post("/api/system/priority-warm")
async def system_priority_warm(
request: Request,
background_tasks: BackgroundTasks,
timezone: Optional[str] = None,
):
_assert_entitlement(request)
batches = _select_priority_city_batches(timezone)
primary = list(batches.get("primary") or [])
secondary = list(batches.get("secondary") or [])
return {
"history": out,
"mode": "full" if include_records else "preview",
"has_more": len(all_days) > len(selected_days),
"full_count": len(all_days),
"preview_count": len(out),
"settlement_source": source,
"settlement_source_label": SETTLEMENT_SOURCE_LABELS.get(source, source.upper()),
}
def _runner() -> None:
for city in primary:
try:
_refresh_city_summary_cache(city, force_refresh=False)
_refresh_city_panel_cache(city, force_refresh=False)
_refresh_city_nearby_cache(city, force_refresh=False)
_refresh_city_market_cache(city, force_refresh=False)
except Exception as exc:
logger.warning("priority warm primary failed city={} timezone={}: {}", city, timezone, exc)
for city in secondary:
try:
_refresh_city_summary_cache(city, force_refresh=False)
_refresh_city_panel_cache(city, force_refresh=False)
except Exception as exc:
logger.warning("priority warm secondary failed city={} timezone={}: {}", city, timezone, exc)
return await run_in_threadpool(_build_history_payload)
background_tasks.add_task(_runner)
return {
"ok": True,
"region": batches.get("region"),
"timezone": batches.get("timezone"),
"primary": primary,
"secondary": secondary,
}
@router.get("/api/auth/me")
@@ -1063,10 +1350,21 @@ async def payment_reconcile_latest(request: Request):
@router.get("/api/city/{name}/summary")
async def city_summary(request: Request, name: str, force_refresh: bool = False):
async def city_summary(
request: Request,
background_tasks: BackgroundTasks,
name: str,
force_refresh: bool = False,
):
city = _normalize_city_or_404(name)
data = await run_in_threadpool(_analyze_summary, city, force_refresh)
return await run_in_threadpool(_build_city_summary_payload, data)
if force_refresh:
return await run_in_threadpool(_refresh_city_summary_cache, city, True)
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "summary", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_SUMMARY_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="summary", city=city)
return cached_entry.get("payload") or {}
return await run_in_threadpool(_refresh_city_summary_cache, city, False)
@router.get("/api/city/{name}/detail")
@@ -1091,6 +1389,7 @@ async def city_detail_aggregate(
@router.get("/api/city/{name}/market-scan")
async def city_market_scan(
request: Request,
background_tasks: BackgroundTasks,
name: str,
force_refresh: bool = False,
market_slug: Optional[str] = None,
@@ -1098,7 +1397,16 @@ async def city_market_scan(
):
_assert_entitlement(request)
city = _normalize_city_or_404(name)
data = await run_in_threadpool(_analyze, city, force_refresh, False, "market")
if force_refresh:
data = await run_in_threadpool(_refresh_city_market_cache, city, True)
else:
cached_entry = await run_in_threadpool(_CACHE_DB.get_city_cache, "market", city)
if cached_entry:
if not _city_cache_is_fresh(cached_entry, CITY_MARKET_CACHE_TTL_SEC):
_schedule_cache_refresh(background_tasks, kind="market", city=city)
data = cached_entry.get("payload") or {}
else:
data = await run_in_threadpool(_refresh_city_market_cache, city, False)
return await run_in_threadpool(
_build_city_market_scan_payload,
data,