Decouple live weather reads from source collection

This commit is contained in:
2569718930@qq.com
2026-06-14 17:53:24 +08:00
parent a098a5ab20
commit e111031119
18 changed files with 2248 additions and 117 deletions
+541
View File
@@ -524,6 +524,111 @@ class DBManager:
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS canonical_temperature_latest (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
value REAL,
source TEXT,
source_role TEXT,
observed_at TEXT,
fetched_at TEXT,
freshness_sec INTEGER,
freshness_status TEXT,
confidence REAL,
explanation TEXT,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_canonical_temperature_latest_updated
ON canonical_temperature_latest(updated_at_ts DESC)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS raw_observation_store (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL,
city TEXT NOT NULL,
station_code TEXT NOT NULL DEFAULT '',
station_name TEXT NOT NULL DEFAULT '',
runway TEXT NOT NULL DEFAULT '',
value REAL,
value_unit TEXT NOT NULL DEFAULT '',
observed_at TEXT,
fetched_at TEXT NOT NULL,
source_latency_sec REAL,
status TEXT NOT NULL DEFAULT 'ok',
error_count INTEGER NOT NULL DEFAULT 0,
last_success_at TEXT,
payload_json TEXT NOT NULL,
created_at_ts REAL NOT NULL
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_raw_observation_store_source_city_time
ON raw_observation_store(source, city, observed_at DESC, fetched_at DESC)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS raw_observation_latest (
source TEXT NOT NULL,
city TEXT NOT NULL,
station_code TEXT NOT NULL DEFAULT '',
station_name TEXT NOT NULL DEFAULT '',
runway TEXT NOT NULL DEFAULT '',
value REAL,
value_unit TEXT NOT NULL DEFAULT '',
observed_at TEXT,
fetched_at TEXT NOT NULL,
source_latency_sec REAL,
status TEXT NOT NULL DEFAULT 'ok',
error_count INTEGER NOT NULL DEFAULT 0,
last_success_at TEXT,
payload_json TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
PRIMARY KEY (source, city, station_code, runway)
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_raw_observation_latest_city_source
ON raw_observation_latest(city, source, updated_at_ts DESC)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS observation_refresh_requests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
city TEXT NOT NULL,
kind TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT '',
priority TEXT NOT NULL DEFAULT 'normal',
reason TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
owner TEXT NOT NULL DEFAULT '',
requested_at TEXT NOT NULL,
requested_at_ts REAL NOT NULL,
claimed_at_ts REAL,
completed_at_ts REAL,
last_error TEXT NOT NULL DEFAULT ''
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_observation_refresh_requests_status
ON observation_refresh_requests(status, priority, requested_at_ts)
"""
)
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_observation_refresh_requests_city
ON observation_refresh_requests(city, kind, status)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS cache_refresh_locks (
cache_key TEXT PRIMARY KEY,
@@ -727,6 +832,24 @@ class DBManager:
conn.commit()
logger.info(f"Database initialized successfully path={self.db_path}")
@staticmethod
def _float_or_none(value: Any) -> Optional[float]:
try:
if value is None or value == "":
return None
return float(value)
except (TypeError, ValueError):
return None
@staticmethod
def _int_or_none(value: Any) -> Optional[int]:
try:
if value is None or value == "":
return None
return int(float(value))
except (TypeError, ValueError):
return None
def _cache_table_name(self, kind: str) -> Optional[str]:
normalized = str(kind or "").strip().lower()
if normalized == "summary":
@@ -819,6 +942,424 @@ class DBManager:
)
conn.commit()
def get_canonical_temperature(self, city: str) -> Optional[Dict[str, Any]]:
normalized_city = str(city or "").strip().lower()
if not normalized_city:
return None
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
row = conn.execute(
"""
SELECT city, payload_json, value, source, source_role, observed_at,
fetched_at, freshness_sec, freshness_status, confidence,
explanation, updated_at, updated_at_ts
FROM canonical_temperature_latest
WHERE city = ?
LIMIT 1
""",
(normalized_city,),
).fetchone()
if not row:
return None
try:
payload = json.loads(str(row["payload_json"] or "{}"))
except Exception:
return None
if not isinstance(payload, dict):
return None
return {
"city": str(row["city"] or normalized_city),
"payload": payload,
"value": self._float_or_none(row["value"]),
"source": str(row["source"] or ""),
"source_role": str(row["source_role"] or ""),
"observed_at": str(row["observed_at"] or ""),
"fetched_at": str(row["fetched_at"] or ""),
"freshness_sec": self._int_or_none(row["freshness_sec"]),
"freshness_status": str(row["freshness_status"] or ""),
"confidence": self._float_or_none(row["confidence"]),
"explanation": str(row["explanation"] or ""),
"updated_at": str(row["updated_at"] or ""),
"updated_at_ts": float(row["updated_at_ts"] or 0.0),
}
def set_canonical_temperature(self, city: str, payload: Dict[str, Any]) -> None:
normalized_city = str(city or "").strip().lower()
if not normalized_city or not isinstance(payload, dict):
return
value = self._float_or_none(payload.get("value"))
source = str(payload.get("source") or "").strip().lower()
source_role = str(payload.get("source_role") or "").strip().lower()
observed_at = str(payload.get("observed_at") or "").strip()
fetched_at = str(payload.get("fetched_at") or "").strip()
freshness_sec = self._int_or_none(payload.get("freshness_sec"))
freshness_status = str(payload.get("freshness_status") or "").strip().lower()
confidence = self._float_or_none(payload.get("confidence"))
explanation = str(payload.get("explanation") or "").strip()
now_dt = datetime.now()
now = now_dt.isoformat()
now_ts = now_dt.timestamp()
with self._get_connection() as conn:
conn.execute(
"""
INSERT INTO canonical_temperature_latest (
city, payload_json, value, source, source_role, observed_at,
fetched_at, freshness_sec, freshness_status, confidence,
explanation, updated_at, updated_at_ts
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(city) DO UPDATE SET
payload_json = excluded.payload_json,
value = excluded.value,
source = excluded.source,
source_role = excluded.source_role,
observed_at = excluded.observed_at,
fetched_at = excluded.fetched_at,
freshness_sec = excluded.freshness_sec,
freshness_status = excluded.freshness_status,
confidence = excluded.confidence,
explanation = excluded.explanation,
updated_at = excluded.updated_at,
updated_at_ts = excluded.updated_at_ts
""",
(
normalized_city,
json.dumps({**payload, "city": normalized_city}, ensure_ascii=False),
value,
source,
source_role,
observed_at,
fetched_at,
freshness_sec,
freshness_status,
confidence,
explanation,
now,
now_ts,
),
)
conn.commit()
def append_raw_observation(
self,
*,
source: str,
city: str,
value: Any = None,
observed_at: str = "",
fetched_at: str = "",
station_code: str = "",
station_name: str = "",
runway: str = "",
value_unit: str = "",
source_latency_sec: Any = None,
status: str = "ok",
error_count: int = 0,
last_success_at: str = "",
payload: Optional[Dict[str, Any]] = None,
) -> None:
normalized_source = str(source or "").strip().lower()
normalized_city = str(city or "").strip().lower()
if not normalized_source or not normalized_city:
return
safe_station_code = str(station_code or "").strip().upper()
safe_station_name = str(station_name or "").strip()
safe_runway = str(runway or "").strip().upper()
safe_observed_at = str(observed_at or "").strip()
now_dt = datetime.now()
safe_fetched_at = str(fetched_at or now_dt.isoformat()).strip()
safe_status = str(status or "ok").strip().lower() or "ok"
value_float = self._float_or_none(value)
latency_float = self._float_or_none(source_latency_sec)
payload_json = json.dumps(payload or {}, ensure_ascii=False)
created_at_ts = now_dt.timestamp()
success_at = str(last_success_at or (safe_fetched_at if safe_status == "ok" else "")).strip()
with self._get_connection() as conn:
conn.execute(
"""
INSERT INTO raw_observation_store (
source, city, station_code, station_name, runway, value,
value_unit, observed_at, fetched_at, source_latency_sec,
status, error_count, last_success_at, payload_json, created_at_ts
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
normalized_source,
normalized_city,
safe_station_code,
safe_station_name,
safe_runway,
value_float,
str(value_unit or "").strip(),
safe_observed_at,
safe_fetched_at,
latency_float,
safe_status,
max(0, int(error_count or 0)),
success_at,
payload_json,
created_at_ts,
),
)
conn.execute(
"""
INSERT INTO raw_observation_latest (
source, city, station_code, station_name, runway, value,
value_unit, observed_at, fetched_at, source_latency_sec,
status, error_count, last_success_at, payload_json, updated_at_ts
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source, city, station_code, runway) DO UPDATE SET
station_name = excluded.station_name,
value = excluded.value,
value_unit = excluded.value_unit,
observed_at = excluded.observed_at,
fetched_at = excluded.fetched_at,
source_latency_sec = excluded.source_latency_sec,
status = excluded.status,
error_count = excluded.error_count,
last_success_at = excluded.last_success_at,
payload_json = excluded.payload_json,
updated_at_ts = excluded.updated_at_ts
""",
(
normalized_source,
normalized_city,
safe_station_code,
safe_station_name,
safe_runway,
value_float,
str(value_unit or "").strip(),
safe_observed_at,
safe_fetched_at,
latency_float,
safe_status,
max(0, int(error_count or 0)),
success_at,
payload_json,
created_at_ts,
),
)
conn.commit()
def get_latest_raw_observation(
self,
source: str,
city: str,
*,
station_code: str = "",
runway: str = "",
) -> Optional[Dict[str, Any]]:
normalized_source = str(source or "").strip().lower()
normalized_city = str(city or "").strip().lower()
if not normalized_source or not normalized_city:
return None
filters = ["source = ?", "city = ?"]
params: List[Any] = [normalized_source, normalized_city]
if station_code:
filters.append("station_code = ?")
params.append(str(station_code or "").strip().upper())
if runway:
filters.append("runway = ?")
params.append(str(runway or "").strip().upper())
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
row = conn.execute(
f"""
SELECT *
FROM raw_observation_latest
WHERE {' AND '.join(filters)}
ORDER BY updated_at_ts DESC
LIMIT 1
""",
params,
).fetchone()
if not row:
return None
try:
payload = json.loads(str(row["payload_json"] or "{}"))
except Exception:
payload = {}
if not isinstance(payload, dict):
payload = {}
return {
"source": str(row["source"] or ""),
"city": str(row["city"] or ""),
"station_code": str(row["station_code"] or ""),
"station_name": str(row["station_name"] or ""),
"runway": str(row["runway"] or ""),
"value": self._float_or_none(row["value"]),
"value_unit": str(row["value_unit"] or ""),
"observed_at": str(row["observed_at"] or ""),
"fetched_at": str(row["fetched_at"] or ""),
"source_latency_sec": self._float_or_none(row["source_latency_sec"]),
"status": str(row["status"] or ""),
"error_count": int(row["error_count"] or 0),
"last_success_at": str(row["last_success_at"] or ""),
"payload": payload,
"updated_at_ts": float(row["updated_at_ts"] or 0.0),
}
def enqueue_observation_refresh_request(
self,
*,
city: str,
kind: str = "",
source: str = "",
priority: str = "normal",
reason: str = "",
) -> bool:
normalized_city = str(city or "").strip().lower()
normalized_kind = str(kind or "").strip().lower()
normalized_source = str(source or "").strip().lower()
normalized_priority = str(priority or "normal").strip().lower()
if normalized_priority not in {"high", "normal", "low"}:
normalized_priority = "normal"
if not normalized_city:
return False
priority_rank = {"low": 0, "normal": 1, "high": 2}
now_dt = datetime.now()
now = now_dt.isoformat()
now_ts = now_dt.timestamp()
with self._get_connection() as conn:
existing = conn.execute(
"""
SELECT id, priority
FROM observation_refresh_requests
WHERE city = ? AND source = ? AND status IN ('pending', 'claimed')
ORDER BY requested_at_ts DESC
LIMIT 1
""",
(normalized_city, normalized_source),
).fetchone()
if existing:
existing_priority = str(existing[1] or "normal").strip().lower()
if priority_rank.get(existing_priority, 1) > priority_rank[normalized_priority]:
normalized_priority = existing_priority
conn.execute(
"""
UPDATE observation_refresh_requests
SET kind = ?, priority = ?, reason = ?, requested_at = ?, requested_at_ts = ?
WHERE id = ?
""",
(
normalized_kind,
normalized_priority,
str(reason or "").strip(),
now,
now_ts,
int(existing[0]),
),
)
else:
conn.execute(
"""
INSERT INTO observation_refresh_requests (
city, kind, source, priority, reason, status,
requested_at, requested_at_ts
)
VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
""",
(
normalized_city,
normalized_kind,
normalized_source,
normalized_priority,
str(reason or "").strip(),
now,
now_ts,
),
)
conn.commit()
return True
def claim_observation_refresh_requests(
self,
*,
limit: int = 20,
owner: str = "",
now_ts: Optional[float] = None,
) -> List[Dict[str, Any]]:
safe_limit = max(1, min(int(limit or 20), 200))
safe_owner = str(owner or "").strip() or secrets.token_hex(6)
claim_ts = float(now_ts if now_ts is not None else datetime.now().timestamp())
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
rows = conn.execute(
"""
SELECT *
FROM observation_refresh_requests
WHERE status = 'pending'
ORDER BY
CASE priority WHEN 'high' THEN 0 WHEN 'normal' THEN 1 ELSE 2 END,
requested_at_ts ASC
LIMIT ?
""",
(safe_limit,),
).fetchall()
ids = [int(row["id"]) for row in rows]
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"""
UPDATE observation_refresh_requests
SET status = 'claimed',
owner = ?,
attempts = attempts + 1,
claimed_at_ts = ?
WHERE id IN ({placeholders})
""",
[safe_owner, claim_ts, *ids],
)
conn.commit()
return [
{
"id": int(row["id"]),
"city": str(row["city"] or ""),
"kind": str(row["kind"] or ""),
"source": str(row["source"] or ""),
"priority": str(row["priority"] or ""),
"reason": str(row["reason"] or ""),
"status": "claimed",
"attempts": int(row["attempts"] or 0) + 1,
"owner": safe_owner,
"requested_at": str(row["requested_at"] or ""),
"requested_at_ts": float(row["requested_at_ts"] or 0.0),
"claimed_at_ts": claim_ts,
"last_error": str(row["last_error"] or ""),
}
for row in rows
]
def mark_observation_refresh_request_done(
self,
request_id: int,
*,
status: str = "done",
error: str = "",
) -> None:
safe_status = str(status or "done").strip().lower()
if safe_status not in {"done", "failed"}:
safe_status = "done"
with self._get_connection() as conn:
conn.execute(
"""
UPDATE observation_refresh_requests
SET status = ?,
completed_at_ts = ?,
last_error = ?
WHERE id = ?
""",
(
safe_status,
datetime.now().timestamp(),
str(error or "").strip(),
int(request_id),
),
)
conn.commit()
def acquire_cache_refresh_lock(
self,
cache_key: str,
+28 -7
View File
@@ -27,6 +27,7 @@ from src.utils.telegram_i18n import (
normalize_push_language as _normalize_push_language,
telegram_push_language as _resolve_telegram_push_language,
)
from web.services.canonical_temperature import build_city_weather_from_canonical
# Forum topic routing: maps city_key -> message_thread_id for the push forum group.
# Created by scripts/create_forum_topics.py, stored in the runtime data dir.
@@ -1504,9 +1505,11 @@ def _cached_payload_observation_epoch(payload: Dict[str, Any]) -> Optional[int]:
airport_current = payload.get("airport_current") or {}
current = payload.get("current") or {}
candidates = [
(payload.get("canonical_temperature") or {}).get("observed_at"),
amos.get("observation_time"),
airport_primary.get("obs_time"),
airport_current.get("obs_time"),
current.get("observed_at"),
current.get("obs_time"),
]
parsed = [
@@ -1574,19 +1577,37 @@ def _read_cached_airport_city_weather(city: str, max_age_sec: Optional[int] = No
return None
def _read_canonical_airport_city_weather(city: str) -> Optional[Dict[str, Any]]:
normalized_city = (city or "").strip().lower()
if not normalized_city:
return None
try:
db = DBManager()
getter = getattr(db, "get_canonical_temperature", None)
if not callable(getter):
return None
row = getter(normalized_city)
except Exception as exc:
logger.debug("airport push canonical latest read failed city={}: {}", normalized_city, exc)
return None
if not isinstance(row, dict):
return None
canonical = row.get("payload") or row
if not isinstance(canonical, dict):
return None
return build_city_weather_from_canonical(normalized_city, canonical)
def _load_airport_city_weather_for_push(city: str) -> Dict[str, Any]:
cached = _read_cached_airport_city_weather(city)
if cached is not None:
return cached
from web.app import _analyze # lazy import - only the bot process needs it
canonical = _read_canonical_airport_city_weather(city)
if canonical is not None:
return canonical
return _analyze(
city,
force_refresh=False,
force_refresh_observations_only=False,
detail_mode="panel",
)
raise RuntimeError(f"no cached city weather for airport push city={city}")
# Per-city temperature window threshold (°C below DEB predicted high)
+445
View File
@@ -0,0 +1,445 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
def _sample_panel_payload() -> dict:
return {
"name": "shanghai",
"temp_symbol": "°C",
"updated_at": "2026-06-14T01:02:03+00:00",
"current": {
"temp": 31.2,
"source_code": "amsc_awos",
"settlement_source": "amsc_awos",
"settlement_source_label": "AMSC AWOS runway-point air temperature",
"station_code": "ZSPD",
"station_name": "Shanghai Pudong",
"observed_at": "2026-06-14T01:01:00+00:00",
"obs_time": "09:01",
"freshness": {
"freshness_status": "fresh",
"age_sec": 63,
"observed_at": "2026-06-14T01:01:00+00:00",
"observed_at_local": "09:01",
},
},
}
def test_db_manager_stores_canonical_temperature_latest(tmp_path):
from src.database.db_manager import DBManager
db = DBManager(str(tmp_path / "polyweather.db"))
canonical = {
"city": "shanghai",
"value": 31.2,
"source": "amsc_awos",
"source_role": "settlement_proxy",
"observed_at": "2026-06-14T01:01:00+00:00",
"fetched_at": "2026-06-14T01:02:03+00:00",
"freshness_sec": 63,
"freshness_status": "fresh",
"confidence": 0.92,
"explanation": "AMSC AWOS runway-point air temperature updated 63s ago.",
}
db.set_canonical_temperature("Shanghai", canonical)
row = db.get_canonical_temperature("shanghai")
assert row is not None
assert row["city"] == "shanghai"
assert row["value"] == 31.2
assert row["source"] == "amsc_awos"
assert row["source_role"] == "settlement_proxy"
assert row["freshness_status"] == "fresh"
assert row["payload"]["observed_at"] == "2026-06-14T01:01:00+00:00"
def test_refresh_city_panel_cache_persists_canonical_temperature(monkeypatch):
import web.services.city_runtime as city_runtime
writes = {}
class FakeDB:
def set_city_cache(self, kind, city, payload, **_kwargs):
writes["city_cache"] = (kind, city, payload)
def set_canonical_temperature(self, city, payload):
writes["canonical"] = (city, payload)
monkeypatch.setattr(city_runtime, "_CACHE_DB", FakeDB())
monkeypatch.setattr(
city_runtime,
"_analyze",
lambda city, force_refresh=False, detail_mode="panel": _sample_panel_payload(),
)
payload = city_runtime._refresh_city_panel_cache("shanghai")
assert payload["canonical_temperature"]["value"] == 31.2
assert payload["canonical_temperature"]["source"] == "amsc_awos"
assert payload["canonical_temperature"]["freshness_sec"] == 63
assert payload["canonical_temperature"]["source_role"] == "settlement_proxy"
assert writes["canonical"][0] == "shanghai"
assert writes["canonical"][1]["observed_at"] == "2026-06-14T01:01:00+00:00"
def test_city_panel_cold_cache_returns_canonical_latest_without_sync_refresh(monkeypatch):
import web.services.city_api as city_api
started = []
class FakeDB:
def get_city_cache(self, kind, city):
assert kind == "panel"
assert city == "shanghai"
return None
def get_canonical_temperature(self, city):
assert city == "shanghai"
return {
"payload": {
"city": "shanghai",
"value": 31.2,
"temp_symbol": "°C",
"source": "amsc_awos",
"source_label": "AMSC AWOS runway-point air temperature",
"source_role": "settlement_proxy",
"observed_at": "2026-06-14T01:01:00+00:00",
"observed_at_local": "09:01",
"freshness_sec": 63,
"freshness_status": "fresh",
"fetched_at": "2026-06-14T01:02:03+00:00",
"confidence": 0.92,
}
}
def fail_refresh(*_args, **_kwargs):
raise AssertionError("cold panel cache must not synchronously refresh when canonical latest exists")
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeDB())
monkeypatch.setattr(city_api.legacy_routes, "_refresh_city_panel_cache", fail_refresh)
monkeypatch.setattr(
city_api,
"_start_city_cache_stale_refresh",
lambda city, kind, refresh_fn: started.append((city, kind, refresh_fn)),
)
payload = asyncio.run(
city_api.get_city_detail_payload(
object(),
"Shanghai",
force_refresh=False,
depth="panel",
)
)
assert payload["current"]["temp"] == 31.2
assert payload["canonical_temperature"]["source"] == "amsc_awos"
assert payload["detail_depth"] == "panel"
assert [(city, kind) for city, kind, _ in started] == [("shanghai", "panel")]
def test_city_full_cold_cache_returns_canonical_latest_without_sync_refresh(monkeypatch):
import web.services.city_api as city_api
started = []
class FakeDB:
def get_city_cache(self, kind, city):
assert kind == "full"
assert city == "shanghai"
return None
def get_canonical_temperature(self, city):
assert city == "shanghai"
return {
"payload": {
"city": "shanghai",
"value": 31.2,
"temp_symbol": "°C",
"source": "amsc_awos",
"source_label": "AMSC AWOS runway-point air temperature",
"source_role": "settlement_proxy",
"observed_at": "2026-06-14T01:01:00+00:00",
"observed_at_local": "09:01",
"freshness_sec": 63,
"freshness_status": "fresh",
"fetched_at": "2026-06-14T01:02:03+00:00",
"confidence": 0.92,
}
}
def fail_refresh(*_args, **_kwargs):
raise AssertionError("cold full cache must not synchronously refresh when canonical latest exists")
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeDB())
monkeypatch.setattr(city_api.legacy_routes, "_refresh_city_full_cache", fail_refresh)
monkeypatch.setattr(city_api, "_start_city_full_stale_refresh", lambda city: started.append(city))
payload = asyncio.run(city_api._get_city_full_data("shanghai", force_refresh=False))
assert payload["current"]["temp"] == 31.2
assert payload["canonical_temperature"]["source"] == "amsc_awos"
assert payload["detail_depth"] == "full"
assert started == ["shanghai"]
def test_city_nearby_and_market_cold_cache_return_canonical_latest_without_sync_refresh(monkeypatch):
import web.services.city_api as city_api
started = []
requested_kinds = []
class FakeDB:
def get_city_cache(self, kind, city):
requested_kinds.append((kind, city))
return None
def get_canonical_temperature(self, city):
assert city == "shanghai"
return {
"payload": {
"city": "shanghai",
"value": 31.2,
"temp_symbol": "°C",
"source": "amsc_awos",
"source_label": "AMSC AWOS runway-point air temperature",
"source_role": "settlement_proxy",
"observed_at": "2026-06-14T01:01:00+00:00",
"observed_at_local": "09:01",
"freshness_sec": 63,
"freshness_status": "fresh",
"fetched_at": "2026-06-14T01:02:03+00:00",
"confidence": 0.92,
}
}
def fail_refresh(*_args, **_kwargs):
raise AssertionError("cold city cache must not synchronously refresh when canonical latest exists")
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeDB())
monkeypatch.setattr(city_api.legacy_routes, "_refresh_city_nearby_cache", fail_refresh)
monkeypatch.setattr(city_api.legacy_routes, "_refresh_city_market_cache", fail_refresh)
monkeypatch.setattr(
city_api,
"_start_city_cache_stale_refresh",
lambda city, kind, refresh_fn: started.append((city, kind, refresh_fn)),
)
nearby = asyncio.run(
city_api.get_city_detail_payload(object(), "Shanghai", force_refresh=False, depth="nearby")
)
market = asyncio.run(
city_api.get_city_detail_payload(object(), "Shanghai", force_refresh=False, depth="market")
)
assert nearby["detail_depth"] == "nearby"
assert market["detail_depth"] == "market"
assert nearby["current"]["temp"] == 31.2
assert market["current"]["temp"] == 31.2
assert [(city, kind) for city, kind, _ in started] == [
("shanghai", "nearby"),
("shanghai", "market"),
]
assert requested_kinds == [("nearby", "shanghai"), ("market", "shanghai")]
def test_force_refresh_panel_returns_canonical_latest_without_waiting_for_sync_refresh(monkeypatch):
import web.services.city_api as city_api
class FakeDB:
def get_city_cache(self, kind, city):
assert kind == "panel"
assert city == "shanghai"
return None
def get_canonical_temperature(self, city):
assert city == "shanghai"
return {
"payload": {
"city": "shanghai",
"value": 31.2,
"temp_symbol": "°C",
"source": "amsc_awos",
"source_label": "AMSC AWOS runway-point air temperature",
"source_role": "settlement_proxy",
"observed_at": "2026-06-14T01:01:00+00:00",
"observed_at_local": "09:01",
"freshness_sec": 63,
"freshness_status": "fresh",
"fetched_at": "2026-06-14T01:02:03+00:00",
"confidence": 0.92,
}
}
def fail_refresh(*_args, **_kwargs):
raise AssertionError("force refresh must not block on sync refresh when canonical latest exists")
city_api._CITY_FORCE_REFRESH_INFLIGHT.clear()
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeDB())
monkeypatch.setattr(city_api.legacy_routes, "_refresh_city_panel_cache", fail_refresh)
payload = asyncio.run(
city_api.get_city_detail_payload(
object(),
"Shanghai",
force_refresh=True,
depth="panel",
)
)
assert payload["current"]["temp"] == 31.2
assert payload["canonical_temperature"]["source"] == "amsc_awos"
assert payload["detail_depth"] == "panel"
def test_city_panel_canonical_fallback_enqueues_collector_refresh_when_queue_exists(monkeypatch):
import web.services.city_api as city_api
enqueued = []
class FakeDB:
def get_city_cache(self, kind, city):
assert kind == "panel"
assert city == "shanghai"
return None
def get_canonical_temperature(self, city):
assert city == "shanghai"
return {
"payload": {
"city": "shanghai",
"value": 31.2,
"temp_symbol": "°C",
"source": "amsc_awos",
"source_label": "AMSC AWOS runway-point air temperature",
"source_role": "settlement_proxy",
"observed_at": "2026-06-14T01:01:00+00:00",
"observed_at_local": "09:01",
"freshness_sec": 63,
"freshness_status": "fresh",
"fetched_at": "2026-06-14T01:02:03+00:00",
"confidence": 0.92,
}
}
def enqueue_observation_refresh_request(self, **kwargs):
enqueued.append(kwargs)
return True
def fail_start(*_args, **_kwargs):
raise AssertionError("canonical fallback should enqueue collector refresh instead of direct stale refresh")
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeDB())
monkeypatch.setattr(city_api, "_start_city_cache_stale_refresh", fail_start)
payload = asyncio.run(
city_api.get_city_detail_payload(
object(),
"Shanghai",
force_refresh=False,
depth="panel",
)
)
assert payload["current"]["temp"] == 31.2
assert enqueued == [
{
"city": "shanghai",
"kind": "panel",
"priority": "high",
"reason": "canonical_fallback",
}
]
def test_city_panel_cold_start_without_canonical_returns_initializing_payload(monkeypatch):
import web.services.city_api as city_api
enqueued = []
class FakeDB:
def get_city_cache(self, kind, city):
assert kind == "panel"
assert city == "shanghai"
return None
def get_canonical_temperature(self, city):
assert city == "shanghai"
return None
def enqueue_observation_refresh_request(self, **kwargs):
enqueued.append(kwargs)
return True
def fail_refresh(*_args, **_kwargs):
raise AssertionError("cold start without canonical must not synchronously refresh")
monkeypatch.setattr(city_api.legacy_routes, "_normalize_city_or_404", lambda name: name.strip().lower())
monkeypatch.setattr(city_api.legacy_routes, "_CACHE_DB", FakeDB())
monkeypatch.setattr(city_api.legacy_routes, "_refresh_city_panel_cache", fail_refresh)
payload = asyncio.run(
city_api.get_city_detail_payload(
object(),
"Shanghai",
force_refresh=False,
depth="panel",
)
)
assert payload["name"] == "shanghai"
assert payload["detail_depth"] == "panel"
assert payload["status"] == "initializing"
assert payload["current"]["temp"] is None
assert enqueued == [
{
"city": "shanghai",
"kind": "panel",
"priority": "high",
"reason": "cold_start",
}
]
def test_dashboard_init_uses_safe_city_detail_path_without_direct_refresh(monkeypatch):
import web.services.dashboard_init_api as dashboard_init_api
calls = []
class FakeRequest:
headers = {}
state = SimpleNamespace()
class FakeDB:
def get_city_cache(self, kind, city):
assert kind == "panel"
assert city == "shanghai"
return None
def fail_refresh(*_args, **_kwargs):
raise AssertionError("dashboard init must not directly refresh city panel cache")
async def fake_get_city_detail_payload(request, name, *, force_refresh=False, depth="panel"):
calls.append((name, force_refresh, depth))
return {"name": name, "status": "initializing"}
monkeypatch.setattr(dashboard_init_api.legacy_routes, "_bind_optional_supabase_identity", lambda request: None)
monkeypatch.setattr(dashboard_init_api, "_build_cities_payload", lambda: {"cities": []})
monkeypatch.setattr(dashboard_init_api, "_resolve_default_city", lambda request: "shanghai")
monkeypatch.setattr(dashboard_init_api.legacy_routes, "_CACHE_DB", FakeDB())
monkeypatch.setattr(dashboard_init_api.legacy_routes, "_refresh_city_panel_cache", fail_refresh)
monkeypatch.setattr(dashboard_init_api, "get_city_detail_payload", fake_get_city_detail_payload, raising=False)
payload = asyncio.run(dashboard_init_api.build_dashboard_init_payload(FakeRequest()))
assert payload["default_city_detail"] == {"name": "shanghai", "status": "initializing"}
assert calls == [("shanghai", False, "panel")]
+51
View File
@@ -0,0 +1,51 @@
from __future__ import annotations
def test_realtime_stream_reads_canonical_latest_without_analysis(monkeypatch):
import web.services.city_realtime_stream as stream
stream._STREAM_BUFFERS.clear()
class FakeDB:
def get_canonical_temperature(self, city):
assert city == "shanghai"
return {
"payload": {
"city": "shanghai",
"value": 30.5,
"source": "amsc_awos",
"source_label": "AMSC AWOS runway-point air temperature",
"observed_at": "2026-06-14T04:00:00+00:00",
"freshness_status": "fresh",
},
}
def get_city_cache(self, kind, city):
assert city == "shanghai"
if kind != "panel":
return None
return {
"payload": {
"probabilities": {
"distribution": [
{"temp": 30.0, "label": "30°C"},
{"temp": 32.0, "label": "32°C"},
]
}
}
}
def fail_analyze(*_args, **_kwargs):
raise AssertionError("realtime stream must read cache/latest instead of _analyze")
monkeypatch.setattr(stream, "_CACHE_DB", FakeDB(), raising=False)
monkeypatch.setattr(stream, "_analyze", fail_analyze)
payload = stream.get_realtime_stream_payload("shanghai")
assert payload["points"][-1] == {
"timestamp": "2026-06-14T04:00:00+00:00",
"temp": 30.5,
"source": "amsc_awos",
}
assert [row["threshold_c"] for row in payload["thresholds"]] == [30.0, 32.0]
+163
View File
@@ -135,6 +135,169 @@ def test_observation_collector_run_due_once_refreshes_panel_cache():
assert refreshed == ["qingdao", "qingdao"]
def test_raw_observation_store_records_latest_observation(tmp_path):
from src.database.db_manager import DBManager
db = DBManager(str(tmp_path / "polyweather.db"))
db.append_raw_observation(
source="amsc_awos",
city="Qingdao",
value=24.0,
observed_at="2026-06-14T01:00:00+00:00",
fetched_at="2026-06-14T01:01:00+00:00",
station_code="ZSQD",
station_name="Qingdao Jiaodong",
status="ok",
payload={"temp_c": 24.0},
)
latest = db.get_latest_raw_observation("amsc_awos", "qingdao")
assert latest is not None
assert latest["source"] == "amsc_awos"
assert latest["city"] == "qingdao"
assert latest["value"] == 24.0
assert latest["observed_at"] == "2026-06-14T01:00:00+00:00"
assert latest["fetched_at"] == "2026-06-14T01:01:00+00:00"
assert latest["station_code"] == "ZSQD"
assert latest["status"] == "ok"
assert latest["payload"]["temp_c"] == 24.0
def test_observation_refresh_request_queue_claims_pending_requests(tmp_path):
from src.database.db_manager import DBManager
db = DBManager(str(tmp_path / "polyweather.db"))
assert db.enqueue_observation_refresh_request(
city="Shanghai",
kind="panel",
priority="high",
reason="cold_canonical_fallback",
)
claimed = db.claim_observation_refresh_requests(limit=5, owner="collector-1", now_ts=1000.0)
assert len(claimed) == 1
request = claimed[0]
assert request["city"] == "shanghai"
assert request["kind"] == "panel"
assert request["priority"] == "high"
assert request["reason"] == "cold_canonical_fallback"
db.mark_observation_refresh_request_done(request["id"], status="done")
assert db.claim_observation_refresh_requests(limit=5, owner="collector-1", now_ts=1001.0) == []
def test_observation_refresh_request_queue_coalesces_city_source_across_kinds(tmp_path):
from src.database.db_manager import DBManager
db = DBManager(str(tmp_path / "polyweather.db"))
assert db.enqueue_observation_refresh_request(
city="Shanghai",
kind="panel",
priority="high",
reason="panel_cold_start",
)
assert db.enqueue_observation_refresh_request(
city="shanghai",
kind="full",
priority="normal",
reason="chart_cold_start",
)
claimed = db.claim_observation_refresh_requests(limit=5, owner="collector-1", now_ts=1000.0)
assert len(claimed) == 1
request = claimed[0]
assert request["city"] == "shanghai"
assert request["kind"] == "full"
assert request["priority"] == "high"
assert request["reason"] == "chart_cold_start"
def test_observation_collector_writes_raw_observation_store(tmp_path):
from src.database.db_manager import DBManager
from web.observation_collector_service import (
ObservationCollector,
ObservationSourceProfile,
)
db = DBManager(str(tmp_path / "polyweather.db"))
class FakeWeather:
def _uses_fahrenheit(self, city):
return False
def _attach_china_amsc_awos_data(self, results, city, use_fahrenheit):
results["amos"] = {
"source": "amsc_awos",
"temp_c": 24.0,
"observation_time": "2026-06-14T01:00:00+00:00",
"icao": "ZSQD",
"station_label": "Qingdao Jiaodong",
}
collector = ObservationCollector(
weather=FakeWeather(),
profiles=[ObservationSourceProfile("amsc_awos", ("qingdao",), 180)],
observation_store=db,
async_cache_refresh=False,
)
assert collector.run_due_once(now_ts=1000.0) == 1
latest = db.get_latest_raw_observation("amsc_awos", "qingdao")
assert latest is not None
assert latest["value"] == 24.0
assert latest["observed_at"] == "2026-06-14T01:00:00+00:00"
assert latest["station_code"] == "ZSQD"
def test_observation_collector_consumes_refresh_request_queue(tmp_path):
from src.database.db_manager import DBManager
from web.observation_collector_service import (
ObservationCollector,
ObservationSourceProfile,
)
db = DBManager(str(tmp_path / "polyweather.db"))
db.enqueue_observation_refresh_request(
city="qingdao",
kind="panel",
priority="high",
reason="canonical_fallback",
)
calls = []
class FakeWeather:
def _uses_fahrenheit(self, city):
return False
def _attach_china_amsc_awos_data(self, results, city, use_fahrenheit):
calls.append(city)
results["amos"] = {
"source": "amsc_awos",
"temp_c": 24.0,
"observation_time": "2026-06-14T01:00:00+00:00",
"icao": "ZSQD",
}
collector = ObservationCollector(
weather=FakeWeather(),
profiles=[ObservationSourceProfile("amsc_awos", ("qingdao",), 180)],
observation_store=db,
async_cache_refresh=False,
)
assert collector.run_due_once(now_ts=100.0) == 1
assert calls == ["qingdao"]
assert db.claim_observation_refresh_requests(limit=5, owner="test", now_ts=101.0) == []
def test_observation_collector_cache_refresh_does_not_block_source_polling():
from web.observation_collector_service import (
ObservationCollector,
+143
View File
@@ -1,5 +1,6 @@
from web.scan_terminal_filters import normalize_scan_terminal_filters
from web import scan_terminal_cache
from web import scan_terminal_service
from web.scan_terminal_metar_gate import _apply_metar_gate_to_row
from web.scan_terminal_payloads import (
build_failed_scan_terminal_payload,
@@ -82,6 +83,36 @@ def test_scan_terminal_prewarm_covers_default_api_limit():
assert 180 in limits
def test_scan_terminal_prewarm_queues_city_refresh_without_analyze(monkeypatch):
enqueued = []
class _DB:
@staticmethod
def enqueue_observation_refresh_request(**kwargs):
enqueued.append(kwargs)
return True
monkeypatch.setattr(scan_terminal_service, "_SCAN_PREWARM_DB", _DB(), raising=False)
monkeypatch.setattr(
scan_terminal_service,
"_analyze",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
AssertionError("scan terminal prewarm must not fetch external sources")
),
raising=False,
)
assert scan_terminal_service._queue_scan_terminal_city_prewarm("shenzhen") == "shenzhen"
assert enqueued == [
{
"city": "shenzhen",
"kind": "panel",
"priority": "normal",
"reason": "scan_terminal_prewarm",
}
]
def test_scan_city_terminal_rows_reuses_persisted_panel_cache(monkeypatch):
payload = {
"display_name": "Paris",
@@ -119,6 +150,118 @@ def test_scan_city_terminal_rows_reuses_persisted_panel_cache(monkeypatch):
assert result["rows"][0]["current_temp"] == 18.0
def test_scan_city_terminal_rows_uses_canonical_without_analyze(monkeypatch):
enqueued = []
class _Cache:
@staticmethod
def get_city_cache(kind, city):
assert (kind, city) == ("panel", "qingdao")
return None
@staticmethod
def get_canonical_temperature(city):
assert city == "qingdao"
return {
"payload": {
"city": "qingdao",
"value": 22.5,
"temp_symbol": "°C",
"source": "amsc_awos",
"source_label": "AMSC AWOS",
"source_role": "settlement_proxy",
"observed_at": "2026-06-01T08:00:00Z",
"fetched_at": "2026-06-01T08:00:30Z",
"freshness_sec": 30,
"freshness_status": "fresh",
"confidence": 0.92,
}
}
@staticmethod
def enqueue_observation_refresh_request(**kwargs):
enqueued.append(kwargs)
return True
monkeypatch.setattr(scan_terminal_city_row, "_PANEL_CACHE_DB", _Cache())
monkeypatch.setattr(
scan_terminal_city_row,
"_analyze",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
AssertionError("scan terminal must not fetch external sources")
),
)
result = scan_terminal_city_row._scan_city_terminal_rows(
"qingdao",
{"market_type": "maxtemp"},
force_refresh=False,
)
assert result["city"] == "qingdao"
assert result["candidate_total"] == 1
assert result["rows"][0]["current_temp"] == 22.5
assert result["rows"][0]["temp_symbol"] == "°C"
assert enqueued == [
{
"city": "qingdao",
"kind": "panel",
"priority": "high",
"reason": "scan_terminal_canonical_fallback",
}
]
def test_scan_city_terminal_rows_cold_start_only_enqueues(monkeypatch):
enqueued = []
class _Cache:
@staticmethod
def get_city_cache(kind, city):
assert (kind, city) == ("panel", "seoul")
return None
@staticmethod
def get_canonical_temperature(city):
assert city == "seoul"
return None
@staticmethod
def enqueue_observation_refresh_request(**kwargs):
enqueued.append(kwargs)
return True
monkeypatch.setattr(scan_terminal_city_row, "_PANEL_CACHE_DB", _Cache())
monkeypatch.setattr(
scan_terminal_city_row,
"_analyze",
lambda *_args, **_kwargs: (_ for _ in ()).throw(
AssertionError("scan terminal must not fetch external sources")
),
)
result = scan_terminal_city_row._scan_city_terminal_rows(
"seoul",
{"market_type": "maxtemp"},
force_refresh=False,
)
assert result == {
"city": "seoul",
"rows": [],
"candidate_total": 0,
"primary_scores": [],
}
assert enqueued == [
{
"city": "seoul",
"kind": "panel",
"priority": "high",
"reason": "scan_terminal_cold_start",
}
]
def test_scan_router_does_not_expose_terminal_ai_endpoint():
routes = {
getattr(route, "path", None): getattr(route, "methods", set())
+47 -17
View File
@@ -1,3 +1,5 @@
import pytest
from src.utils.telegram_push import (
HIGH_FREQ_AIRPORT_CITIES,
HIGH_FREQ_AIRPORT_ICAO,
@@ -294,36 +296,64 @@ def test_airport_push_prefers_cache_with_newest_observation(monkeypatch):
assert city_weather["airport_primary"]["temp"] == 31.4
def test_airport_push_fallback_analysis_does_not_force_observation_refresh(monkeypatch):
def test_airport_push_without_cache_uses_canonical_latest_not_analysis(monkeypatch):
import src.utils.telegram_push as telegram_push
import web.app as web_app
calls = []
class FakeDB:
def get_city_cache(self, kind, city):
return None
def fake_analyze(city, force_refresh=False, force_refresh_observations_only=False, detail_mode="full", **_kwargs):
calls.append((city, force_refresh, force_refresh_observations_only, detail_mode))
return {
"local_time": "12:00",
"current": {"temp": 31.0},
"deb": {"prediction": 29.0},
"airport_current": {"max_so_far": 30.0, "max_temp_time": "11:50", "obs_time": "12:00"},
"mgm_nearby": [
{"icao": "ZSQD", "temp": 31.0, "obs_time": "2026-05-17T04:00:00Z"},
],
}
def get_canonical_temperature(self, city):
assert city == "qingdao"
return {
"payload": {
"city": "qingdao",
"value": 31.0,
"source": "amsc_awos",
"source_label": "AMSC AWOS runway-point air temperature",
"source_role": "settlement_proxy",
"observed_at": "2026-06-14T04:00:00+00:00",
"observed_at_local": "12:00",
"freshness_sec": 45,
"freshness_status": "fresh",
"fetched_at": "2026-06-14T04:00:45+00:00",
"confidence": 0.92,
},
}
def fail_analyze(*_args, **_kwargs):
raise AssertionError("Telegram push must not call _analyze when canonical latest exists")
monkeypatch.setattr(telegram_push, "DBManager", lambda: FakeDB())
monkeypatch.setattr(web_app, "_analyze", fake_analyze)
monkeypatch.setattr(web_app, "_analyze", fail_analyze)
city_weather = telegram_push._load_airport_city_weather_for_push("qingdao")
assert city_weather["current"]["temp"] == 31.0
assert ("qingdao", False, False, "panel") in calls
assert not any(city == "qingdao" and force_obs for city, _force, force_obs, _mode in calls)
assert city_weather["airport_primary"]["obs_time"] == "2026-06-14T04:00:00+00:00"
assert city_weather["current"]["freshness"]["freshness_status"] == "fresh"
def test_airport_push_without_cache_or_canonical_skips_analysis(monkeypatch):
import src.utils.telegram_push as telegram_push
import web.app as web_app
class FakeDB:
def get_city_cache(self, kind, city):
return None
def get_canonical_temperature(self, city):
return None
def fail_analyze(*_args, **_kwargs):
raise AssertionError("Telegram push must not call _analyze on cold cache")
monkeypatch.setattr(telegram_push, "DBManager", lambda: FakeDB())
monkeypatch.setattr(web_app, "_analyze", fail_analyze)
with pytest.raises(RuntimeError, match="no cached city weather"):
telegram_push._load_airport_city_weather_for_push("qingdao")
def test_airport_push_uses_stale_cache_before_fallback_analysis(monkeypatch):
+76 -22
View File
@@ -129,6 +129,69 @@ def test_system_cache_status_requires_ops_admin(monkeypatch):
assert response.status_code in {401, 403, 503}
def test_system_priority_warm_enqueues_collector_refresh_without_direct_refresh(monkeypatch):
from web.services import system_api
tasks = []
enqueued = []
class _BackgroundTasks:
@staticmethod
def add_task(fn, *args, **kwargs):
tasks.append((fn, args, kwargs))
class _Cache:
@staticmethod
def enqueue_observation_refresh_request(**kwargs):
enqueued.append(kwargs)
return True
def fail_refresh(*_args, **_kwargs):
raise AssertionError("priority warm must enqueue collector refreshes")
monkeypatch.setattr(system_api.legacy_routes, "_assert_entitlement", lambda request: None)
monkeypatch.setattr(
system_api.legacy_routes,
"_select_priority_city_batches",
lambda timezone: {
"region": "asia",
"timezone": timezone,
"primary": ["shenzhen"],
"secondary": ["seoul"],
},
)
monkeypatch.setattr(system_api.legacy_routes, "_CACHE_DB", _Cache())
monkeypatch.setattr(system_api.legacy_routes, "_refresh_city_summary_cache", fail_refresh)
monkeypatch.setattr(system_api.legacy_routes, "_refresh_city_panel_cache", fail_refresh)
monkeypatch.setattr(system_api.legacy_routes, "_refresh_city_nearby_cache", fail_refresh)
monkeypatch.setattr(system_api.legacy_routes, "_refresh_city_market_cache", fail_refresh)
monkeypatch.setattr(system_api.legacy_routes, "_refresh_city_full_cache", fail_refresh)
payload = system_api.run_system_priority_warm(
object(),
_BackgroundTasks(),
timezone="Asia/Shanghai",
)
tasks[0][0](*tasks[0][1], **tasks[0][2])
assert payload["primary"] == ["shenzhen"]
assert payload["secondary"] == ["seoul"]
assert enqueued == [
{
"city": "shenzhen",
"kind": "panel",
"priority": "high",
"reason": "system_priority_warm",
},
{
"city": "seoul",
"kind": "panel",
"priority": "normal",
"reason": "system_priority_warm",
},
]
def test_standard_growth_funnel_events_are_trackable():
assert {
"landing_view",
@@ -1531,7 +1594,7 @@ def test_concurrent_city_detail_batch_requests_share_inflight_response(monkeypat
assert build_calls == 1
def test_concurrent_city_detail_requests_share_same_full_cache_refresh(monkeypatch):
def test_concurrent_cold_city_detail_requests_return_initializing_without_full_refresh(monkeypatch):
import asyncio
refresh_calls = 0
@@ -1587,8 +1650,9 @@ def test_concurrent_city_detail_requests_share_same_full_cache_refresh(monkeypat
results = asyncio.run(run_two_requests())
assert [item["city"] for item in results] == ["paris", "paris"]
assert refresh_calls == 1
assert build_calls == 1
assert [item["status"] for item in results] == ["initializing", "initializing"]
assert refresh_calls == 0
assert build_calls == 0
def test_stale_city_detail_uses_cached_full_payload_while_refreshing(monkeypatch):
@@ -1887,22 +1951,11 @@ def test_force_refresh_full_detail_returns_cached_payload_when_refresh_is_slow(m
assert refresh_calls == 1
def test_force_refresh_invalidates_short_city_detail_payload_cache(monkeypatch):
def test_force_refresh_cold_city_detail_returns_initializing_without_full_refresh(monkeypatch):
import asyncio
build_calls = 0
refreshed_payloads = [
{
"city": "paris",
"local_date": "2026-05-30",
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [20.0]},
},
{
"city": "paris",
"local_date": "2026-05-30",
"hourly": {"times": ["2026-05-30T00:00:00Z"], "temps": [21.0]},
},
]
refresh_calls = 0
class FakeCache:
def get_city_cache(self, kind, city):
@@ -1913,9 +1966,9 @@ def test_force_refresh_invalidates_short_city_detail_payload_cache(monkeypatch):
return fn(*args, **kwargs)
def refresh_full(city, force_refresh):
assert city == "paris"
assert refreshed_payloads
return refreshed_payloads.pop(0)
nonlocal refresh_calls
refresh_calls += 1
raise AssertionError("cold force refresh must not synchronously refresh full city data")
def build_detail(data, market_slug, target_date, resolution):
nonlocal build_calls
@@ -1947,9 +2000,10 @@ def test_force_refresh_invalidates_short_city_detail_payload_cache(monkeypatch):
),
)
assert first["live_temp"] == 20.0
assert second["live_temp"] == 21.0
assert build_calls == 2
assert first["status"] == "initializing"
assert second["status"] == "initializing"
assert refresh_calls == 0
assert build_calls == 0
def test_payment_runtime_endpoint_returns_shape():
+5
View File
@@ -1820,6 +1820,8 @@ def _analyze(
"settlement_source_label": current_source_label,
"station_code": current_station_code,
"station_name": current_station_name,
"observed_at": current_obs_raw,
"observed_at_local": obs_time_str,
"obs_time": obs_time_str,
"obs_age_min": None if use_settlement_current else metar_age_min,
"freshness": current_freshness,
@@ -2316,8 +2318,11 @@ def _analyze_summary(city: str, force_refresh: bool = False) -> Dict[str, Any]:
"max_so_far": _sf(display_settlement_max),
"max_temp_time": max_temp_time,
"wu_settlement": _sf(wu_settle),
"source_code": current_source,
"settlement_source": current_source,
"settlement_source_label": current_source_label,
"observed_at": obs_t or None,
"observed_at_local": obs_time_str or None,
"obs_time": obs_time_str or None,
"obs_age_min": obs_age_min,
"observation_status": "live" if cur_temp is not None else "missing",
+171 -4
View File
@@ -15,6 +15,7 @@ from src.data_collection.amos_station_sources import AMOS_AIRPORT_CODES
from src.data_collection.amsc_awos_sources import AMSC_AWOS_AIRPORTS
from src.data_collection.city_registry import CITY_REGISTRY
from src.data_collection.hko_obs_sources import HKO_STATIONS
from src.database.db_manager import DBManager
from src.database.runtime_state import ObservationCollectorStatusRepository
@@ -54,6 +55,7 @@ class ObservationCollector:
profiles: Sequence[ObservationSourceProfile],
cache_refresher: Optional[Callable[[str], Any]] = None,
status_recorder: Optional[ObservationCollectorStatusRepository] = None,
observation_store: Optional[Any] = None,
async_cache_refresh: Optional[bool] = None,
cache_refresh_workers: Optional[int] = None,
) -> None:
@@ -61,6 +63,7 @@ class ObservationCollector:
self.profiles = list(profiles)
self.cache_refresher = cache_refresher
self.status_recorder = status_recorder
self.observation_store = observation_store
self._last_run_ts: dict[tuple[str, str], float] = {}
self._lock = threading.Lock()
self._cache_refresh_lock = threading.Lock()
@@ -89,7 +92,7 @@ class ObservationCollector:
def run_due_once(self, *, now_ts: Optional[float] = None) -> int:
now = float(time.time() if now_ts is None else now_ts)
due: List[tuple[ObservationSourceProfile, str]] = []
due: List[tuple[ObservationSourceProfile, str, Optional[int]]] = []
with self._lock:
for profile in self.profiles:
interval = max(1, int(profile.interval_sec or 60))
@@ -98,10 +101,12 @@ class ObservationCollector:
last_ts = float(self._last_run_ts.get(key) or 0.0)
if now - last_ts >= interval:
self._last_run_ts[key] = now
due.append((profile, city))
due.append((profile, city, None))
due.extend(self._claim_due_refresh_requests(now))
completed = 0
for profile, city in due:
for profile, city, request_id in due:
started_wall = time.time()
started_ts = now if now_ts is not None else started_wall
ok = False
@@ -132,8 +137,71 @@ class ObservationCollector:
ok=ok,
error=error,
)
self._mark_refresh_request_done(request_id, ok=ok, error=error)
return completed
def _claim_due_refresh_requests(self, now: float) -> List[tuple[ObservationSourceProfile, str, Optional[int]]]:
store = self.observation_store
claimer = getattr(store, "claim_observation_refresh_requests", None)
if not callable(claimer):
return []
try:
requests = claimer(limit=50, owner="observation_collector", now_ts=now)
except Exception as exc:
logger.debug("observation refresh request claim skipped: {}", exc)
return []
due: List[tuple[ObservationSourceProfile, str, Optional[int]]] = []
for request in requests or []:
if not isinstance(request, dict):
continue
city = str(request.get("city") or "").strip().lower()
requested_source = str(request.get("source") or "").strip().lower()
request_id = int(request.get("id") or 0) or None
matched = False
rate_limited = False
for profile in self.profiles:
profile_source = str(profile.source or "").strip().lower()
if requested_source and requested_source != profile_source:
continue
if city not in set(profile.cities):
continue
key = (profile.source, city)
interval = max(1, int(profile.interval_sec or 60))
with self._lock:
last_ts = float(self._last_run_ts.get(key) or 0.0)
if last_ts and now - last_ts < interval:
rate_limited = True
continue
self._last_run_ts[key] = now
due.append((profile, city, request_id))
matched = True
if not matched:
reason = "rate_limited" if rate_limited else "no_matching_profile"
self._mark_refresh_request_done(request_id, ok=False, error=reason)
return due
def _mark_refresh_request_done(
self,
request_id: Optional[int],
*,
ok: bool,
error: Optional[str],
) -> None:
if not request_id:
return
store = self.observation_store
marker = getattr(store, "mark_observation_refresh_request_done", None)
if not callable(marker):
return
try:
marker(
request_id,
status="done" if ok else "failed",
error="" if ok else str(error or "collection_failed"),
)
except Exception as exc:
logger.debug("observation refresh request completion skipped id={}: {}", request_id, exc)
def _record_source_status(
self,
*,
@@ -187,7 +255,104 @@ class ObservationCollector:
else:
logger.debug("observation collector skipped unknown source={}", normalized_source)
return False
return bool(results)
ok = bool(results)
if ok:
self._store_raw_observations(normalized_source, normalized_city, results)
return ok
@staticmethod
def _observation_value(row: dict[str, Any]) -> Optional[float]:
for key in ("temp_c", "temperature_c", "temp", "value"):
try:
value = row.get(key)
if value is not None and value != "":
return float(value)
except (TypeError, ValueError):
continue
current = row.get("current")
if isinstance(current, dict):
try:
value = current.get("temp")
if value is not None and value != "":
return float(value)
except (TypeError, ValueError):
return None
return None
@staticmethod
def _observation_time(row: dict[str, Any]) -> str:
for key in ("observation_time", "observed_at", "obs_time", "time_utc", "time"):
value = str(row.get(key) or "").strip()
if value:
return value
return ""
@staticmethod
def _station_code(row: dict[str, Any]) -> str:
for key in ("station_code", "icao", "istNo", "station_id", "code"):
value = str(row.get(key) or "").strip()
if value:
return value
return ""
@staticmethod
def _station_name(row: dict[str, Any]) -> str:
for key in ("station_name", "station_label", "name", "label"):
value = str(row.get(key) or "").strip()
if value:
return value
return ""
def _iter_raw_observation_rows(
self,
source: str,
results: dict[str, Any],
) -> Iterable[dict[str, Any]]:
for value in (results or {}).values():
if isinstance(value, dict):
yield value
continue
if isinstance(value, list):
for item in value:
if isinstance(item, dict):
yield item
def _store_raw_observations(self, source: str, city: str, results: dict[str, Any]) -> None:
store = self.observation_store
writer = getattr(store, "append_raw_observation", None)
if not callable(writer):
return
fetched_at = time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime())
wrote = 0
for row in self._iter_raw_observation_rows(source, results):
value = self._observation_value(row)
if value is None:
continue
payload_source = str(row.get("source") or row.get("source_code") or source).strip().lower()
try:
writer(
source=payload_source or source,
city=city,
value=value,
observed_at=self._observation_time(row),
fetched_at=fetched_at,
station_code=self._station_code(row),
station_name=self._station_name(row),
runway=str(row.get("runway") or "").strip(),
value_unit=str(row.get("unit") or row.get("temp_unit") or "c").strip().lower(),
status="ok",
payload=dict(row),
)
wrote += 1
except Exception as exc:
logger.debug(
"raw observation store write skipped source={} city={}: {}",
source,
city,
exc,
)
if wrote:
logger.debug("raw observations stored source={} city={} count={}", source, city, wrote)
def _refresh_city_cache(self, city: str) -> None:
if not callable(self.cache_refresher):
@@ -276,6 +441,7 @@ def start_observation_collector_loop(
cache_refresher: Optional[Callable[[str], Any]] = None,
profiles: Optional[Sequence[ObservationSourceProfile]] = None,
status_recorder: Optional[ObservationCollectorStatusRepository] = None,
observation_store: Optional[Any] = None,
) -> Optional[threading.Thread]:
if not _env_bool("POLYWEATHER_OBSERVATION_COLLECTOR_ENABLED", True):
return None
@@ -290,6 +456,7 @@ def start_observation_collector_loop(
profiles=selected_profiles,
cache_refresher=cache_refresher,
status_recorder=status_recorder or ObservationCollectorStatusRepository(),
observation_store=observation_store or DBManager(),
)
global _COLLECTOR_THREAD
+50 -12
View File
@@ -7,17 +7,18 @@ from typing import Any, Dict, List, Optional
from src.database.db_manager import DBManager
from web.core import CITIES, _sf as _safe_float
from web.analysis_service import _analyze
from web.scan_terminal_filters import (
market_region_from_tz_offset as _market_region_from_tz_offset,
safe_int as _safe_int,
)
from web.services.canonical_temperature import build_city_weather_from_canonical
from web.services.city_payloads import aggregate_runway_history
SCAN_ROW_RUNWAY_HISTORY_RESOLUTION = "10m"
SCAN_ROW_MAX_RUNWAY_POINTS = 144
_PANEL_CACHE_DB = DBManager()
_analyze = None # compatibility hook for tests that assert scan terminal stays cache-only.
def _compact_runway_plate_history_for_scan(raw_history: Any) -> Dict[str, List[Dict[str, Any]]]:
@@ -31,6 +32,47 @@ def _compact_runway_plate_history_for_scan(raw_history: Any) -> Dict[str, List[D
}
def _enqueue_scan_terminal_refresh(city: str, *, reason: str) -> None:
enqueue = getattr(_PANEL_CACHE_DB, "enqueue_observation_refresh_request", None)
if not callable(enqueue):
return
try:
enqueue(
city=city,
kind="panel",
priority="high",
reason=reason,
)
except Exception:
return
def _load_scan_panel_payload(city: str, *, force_refresh: bool) -> Optional[Dict[str, Any]]:
if not force_refresh:
cached_entry = _PANEL_CACHE_DB.get_city_cache("panel", city)
cached_payload = cached_entry.get("payload") if isinstance(cached_entry, dict) else None
if isinstance(cached_payload, dict):
return cached_payload
canonical_getter = getattr(_PANEL_CACHE_DB, "get_canonical_temperature", None)
canonical_entry = canonical_getter(city) if callable(canonical_getter) else None
canonical = (
canonical_entry.get("payload")
if isinstance(canonical_entry, dict) and isinstance(canonical_entry.get("payload"), dict)
else canonical_entry
)
payload = build_city_weather_from_canonical(city, canonical) if isinstance(canonical, dict) else None
if payload:
city_meta = CITIES.get(city) or {}
payload.setdefault("display_name", city_meta.get("name") or city_meta.get("display_name") or city)
payload.setdefault("temp_symbol", canonical.get("temp_symbol") or "°C")
_enqueue_scan_terminal_refresh(city, reason="scan_terminal_canonical_fallback")
return payload
_enqueue_scan_terminal_refresh(city, reason="scan_terminal_cold_start")
return None
def _resolve_time_range_dates(data: Dict[str, Any], time_range: str) -> List[str]:
local_date = str(data.get("local_date") or "").strip()
multi_model_daily = data.get("multi_model_daily") or {}
@@ -173,18 +215,14 @@ def _scan_city_terminal_rows_quick(
) -> Dict[str, Any]:
"""Fast path that returns cached analysis rows only — returns a single row per city
with cached analysis data (Obs, DEB, probabilities) but no market prices."""
data: Dict[str, Any] = {}
if not force_refresh:
cached_entry = _PANEL_CACHE_DB.get_city_cache("panel", city)
cached_payload = cached_entry.get("payload") if isinstance(cached_entry, dict) else None
if isinstance(cached_payload, dict):
data = cached_payload
data = _load_scan_panel_payload(city, force_refresh=force_refresh)
if not data:
data = _analyze(
city,
force_refresh=force_refresh,
detail_mode="panel",
)
return {
"city": city,
"rows": [],
"candidate_total": 0,
"primary_scores": [],
}
row = _build_quick_row(city=city, data=data)
return {
"city": city,
+15 -4
View File
@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional
from loguru import logger
from web.analysis_service import _analyze
from src.database.db_manager import DBManager
from web.core import CITIES
from web.services.scan_terminal_config import (
SCAN_TERMINAL_BUILD_TIMEOUT_SEC,
@@ -43,6 +43,7 @@ from web.scan_terminal_ranker import build_ranked_scan_terminal_result
_SCAN_TERMINAL_INFLIGHT_BUILD_LOCK = threading.Lock()
_SCAN_TERMINAL_INFLIGHT_BUILDS: Dict[str, Future] = {}
_SCAN_PREWARM_DB = DBManager()
SCAN_TERMINAL_STALE_SUCCESS_MAX_AGE_SEC = max(
SCAN_TERMINAL_PAYLOAD_TTL_SEC * 2,
600,
@@ -460,10 +461,20 @@ def _warm_scan_terminal_payloads() -> int:
return ok
def _queue_scan_terminal_city_prewarm(city: str) -> str:
_SCAN_PREWARM_DB.enqueue_observation_refresh_request(
city=city,
kind="panel",
priority="normal",
reason="scan_terminal_prewarm",
)
return city
def start_scan_terminal_prewarm() -> None:
"""Warm analysis caches for all cities at startup so the first terminal
scan returns quickly instead of forcing every city through a cold
_analyze() path.
scan returns quickly without forcing every city through a cold external
source refresh path.
Runs once per process. Safe to call from any thread and at any point
during the server lifecycle.
@@ -483,7 +494,7 @@ def start_scan_terminal_prewarm() -> None:
def _warm_one(city: str) -> str:
try:
_analyze(city, force_refresh=False, detail_mode="panel")
_queue_scan_terminal_city_prewarm(city)
except Exception:
pass
return city
+214
View File
@@ -0,0 +1,214 @@
from __future__ import annotations
from copy import deepcopy
from datetime import datetime, timezone
from typing import Any, Dict, Optional
_SETTLEMENT_PROXY_SOURCES = {"amsc_awos", "amos", "runway", "awos"}
_SETTLEMENT_OFFICIAL_SOURCES = {"hko", "cwa", "noaa", "wunderground", "mgm", "knmi", "ims"}
_AIRPORT_OFFICIAL_SOURCES = {"metar", "madis_hfmetar", "aeroweb"}
def _to_float(value: Any) -> Optional[float]:
try:
if value is None or value == "":
return None
return float(value)
except (TypeError, ValueError):
return None
def _to_int(value: Any) -> Optional[int]:
try:
if value is None or value == "":
return None
return int(float(value))
except (TypeError, ValueError):
return None
def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _source_role(source: str) -> str:
normalized = str(source or "").strip().lower()
if normalized in _SETTLEMENT_PROXY_SOURCES or "amsc" in normalized or "runway" in normalized:
return "settlement_proxy"
if normalized in _SETTLEMENT_OFFICIAL_SOURCES:
return "settlement_official"
if normalized in _AIRPORT_OFFICIAL_SOURCES:
return "airport_official"
if "model" in normalized or normalized in {"deb", "open_meteo"}:
return "model_blend"
return "fallback"
def _confidence(source_role: str, freshness_status: str) -> float:
base = {
"settlement_proxy": 0.92,
"settlement_official": 0.9,
"airport_official": 0.78,
"model_blend": 0.55,
"fallback": 0.42,
}.get(source_role, 0.42)
penalty = {
"fresh": 0.0,
"expected_wait": 0.03,
"delayed": 0.1,
"stale": 0.18,
"expired": 0.28,
"missing": 0.35,
"unknown": 0.12,
}.get(str(freshness_status or "").strip().lower(), 0.12)
return round(max(0.05, min(0.99, base - penalty)), 2)
def build_canonical_temperature(
city: str,
payload: Dict[str, Any],
*,
fetched_at: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
if not isinstance(payload, dict):
return None
current = payload.get("current") or {}
if not isinstance(current, dict):
return None
value = _to_float(current.get("temp"))
if value is None:
return None
freshness = current.get("freshness") or {}
if not isinstance(freshness, dict):
freshness = {}
source = str(
current.get("source_code")
or current.get("settlement_source")
or current.get("source")
or freshness.get("source_code")
or ""
).strip().lower()
source_label = str(
current.get("settlement_source_label")
or current.get("source_label")
or freshness.get("source_label")
or source.upper()
).strip()
observed_at = str(
current.get("observed_at")
or current.get("observation_time")
or freshness.get("observed_at")
or ""
).strip()
observed_at_local = str(
current.get("observed_at_local")
or freshness.get("observed_at_local")
or current.get("obs_time")
or ""
).strip()
freshness_sec = _to_int(freshness.get("age_sec"))
if freshness_sec is None:
age_min = _to_float(current.get("obs_age_min"))
if age_min is not None:
freshness_sec = int(age_min * 60)
freshness_status = str(
freshness.get("freshness_status")
or current.get("observation_status")
or "unknown"
).strip().lower()
role = _source_role(source)
canonical = {
"city": str(city or payload.get("name") or payload.get("city") or "").strip().lower(),
"value": round(value, 2),
"temp_symbol": str(payload.get("temp_symbol") or "°C"),
"source": source,
"source_label": source_label,
"source_role": role,
"station_code": current.get("station_code"),
"station_name": current.get("station_name"),
"observed_at": observed_at or None,
"observed_at_local": observed_at_local or None,
"fetched_at": str(fetched_at or payload.get("updated_at") or _now_iso()),
"freshness_sec": freshness_sec,
"freshness_status": freshness_status,
"confidence": _confidence(role, freshness_status),
}
age_text = f" updated {freshness_sec}s ago" if freshness_sec is not None else ""
canonical["explanation"] = f"{source_label or source or 'Source'}{age_text}."
return canonical
def attach_canonical_temperature(
payload: Dict[str, Any],
*,
city: Optional[str] = None,
) -> Dict[str, Any]:
if not isinstance(payload, dict):
return payload
next_payload = payload
canonical = build_canonical_temperature(
city or str(payload.get("name") or payload.get("city") or ""),
next_payload,
)
if canonical:
next_payload["canonical_temperature"] = canonical
return next_payload
def store_canonical_temperature_from_payload(db: Any, city: str, payload: Dict[str, Any]) -> Optional[Dict[str, Any]]:
canonical = build_canonical_temperature(city, payload)
if not canonical:
return None
setter = getattr(db, "set_canonical_temperature", None)
if callable(setter):
setter(city, canonical)
return canonical
def build_city_weather_from_canonical(city: str, canonical: Dict[str, Any]) -> Optional[Dict[str, Any]]:
if not isinstance(canonical, dict):
return None
value = _to_float(canonical.get("value"))
if value is None:
return None
observed_at = str(canonical.get("observed_at") or "").strip()
observed_at_local = str(canonical.get("observed_at_local") or "").strip()
source = str(canonical.get("source") or "").strip().lower()
source_label = str(canonical.get("source_label") or source.upper()).strip()
freshness = {
"freshness_status": canonical.get("freshness_status") or "unknown",
"age_sec": canonical.get("freshness_sec"),
"observed_at": observed_at or None,
"observed_at_local": observed_at_local or None,
}
current = {
"temp": value,
"source_code": source,
"settlement_source": source,
"settlement_source_label": source_label,
"observed_at": observed_at or None,
"observed_at_local": observed_at_local or None,
"obs_time": observed_at_local or observed_at,
"freshness": freshness,
"observation_status": "live",
}
airport_primary = {
"temp": value,
"source_code": source,
"source_label": source_label,
"obs_time": observed_at or observed_at_local,
"freshness": freshness,
}
payload = {
"name": str(city or canonical.get("city") or "").strip().lower(),
"temp_symbol": str(canonical.get("temp_symbol") or "°C"),
"current": current,
"airport_primary": airport_primary,
"airport_current": deepcopy(airport_primary),
"canonical_temperature": dict(canonical),
"deb": {"prediction": canonical.get("deb_prediction")},
"updated_at": canonical.get("fetched_at"),
}
return payload
+195 -13
View File
@@ -15,6 +15,7 @@ from loguru import logger
import web.routes as legacy_routes
from web.analysis_service import _runway_history_temp_for_city
from web.services.canonical_temperature import build_city_weather_from_canonical
from web.services.request_timing import ServerTimingRecorder
_RECENT_DEB_CACHE: Optional[Dict[str, Dict[str, object]]] = None
@@ -129,6 +130,132 @@ async def _get_cached_city_payload(city: str, kind: str) -> Dict[str, Any]:
return payload if isinstance(payload, dict) else {}
async def _get_canonical_city_payload(city: str, *, detail_depth: str = "panel") -> Dict[str, Any]:
try:
row = await run_in_threadpool(legacy_routes._CACHE_DB.get_canonical_temperature, city)
except Exception:
return {}
if not isinstance(row, dict):
return {}
canonical = row.get("payload") or row
if not isinstance(canonical, dict):
return {}
payload = build_city_weather_from_canonical(city, canonical)
if not isinstance(payload, dict) or not payload:
return {}
city_meta = legacy_routes.CITY_REGISTRY.get(city, {}) or {}
city_info = legacy_routes.CITIES.get(city, {}) or {}
risk = legacy_routes.CITY_RISK_PROFILES.get(city, {}) or {}
payload.update(
{
"detail_depth": detail_depth,
"display_name": str(city_meta.get("display_name") or city_meta.get("name") or city.title()),
"lat": city_info.get("lat"),
"lon": city_info.get("lon"),
"temp_symbol": canonical.get("temp_symbol") or payload.get("temp_symbol") or ("°F" if city_info.get("f") else "°C"),
"risk": {
"level": risk.get("risk_level", "low"),
"emoji": risk.get("risk_emoji", "🟢"),
"airport": risk.get("airport_name", ""),
"icao": risk.get("icao", ""),
"distance_km": risk.get("distance_km", 0),
"warning": risk.get("warning", ""),
},
"probabilities": {"mu": None, "distribution": []},
}
)
return payload
def _enqueue_collector_refresh_request(
city: str,
kind: str,
*,
reason: str = "canonical_fallback",
) -> bool:
try:
enqueue = getattr(legacy_routes._CACHE_DB, "enqueue_observation_refresh_request", None)
if not callable(enqueue):
return False
return bool(
enqueue(
city=city,
kind=kind,
priority="high",
reason=reason,
)
)
except Exception as exc:
logger.debug("collector refresh enqueue failed city={} kind={}: {}", city, kind, exc)
return False
def _request_city_cache_refresh(
city: str,
kind: str,
refresh_fn: Callable[[str, bool], Dict[str, Any]],
) -> None:
if _enqueue_collector_refresh_request(city, kind):
return
_start_city_cache_stale_refresh(city, kind, refresh_fn)
def _request_city_full_refresh(city: str) -> None:
if _enqueue_collector_refresh_request(city, "full"):
return
_start_city_full_stale_refresh(city)
def _build_initializing_city_payload(city: str, *, detail_depth: str) -> Dict[str, Any]:
city_meta = legacy_routes.CITY_REGISTRY.get(city, {}) or {}
city_info = legacy_routes.CITIES.get(city, {}) or {}
risk = legacy_routes.CITY_RISK_PROFILES.get(city, {}) or {}
return {
"city": city,
"name": city,
"display_name": str(city_meta.get("display_name") or city_meta.get("name") or city.title()),
"detail_depth": detail_depth,
"status": "initializing",
"stale": True,
"stale_reason": "collector_refresh_queued",
"lat": city_info.get("lat"),
"lon": city_info.get("lon"),
"temp_symbol": "°F" if city_info.get("f") else "°C",
"risk": {
"level": risk.get("risk_level", "low"),
"emoji": risk.get("risk_emoji", "🟢"),
"airport": risk.get("airport_name", ""),
"icao": risk.get("icao", ""),
"distance_km": risk.get("distance_km", 0),
"warning": risk.get("warning", ""),
},
"current": {
"temp": None,
"source_code": None,
"settlement_source": None,
"settlement_source_label": None,
"obs_time": None,
"freshness": {
"freshness_status": "missing",
"freshness_reason": "collector_refresh_queued",
},
"observation_status": "initializing",
},
"airport_current": {},
"airport_primary": {},
"canonical_temperature": None,
"deb": {"prediction": None},
"probabilities": {"mu": None, "distribution": []},
"hourly": {"times": [], "temps": []},
"multi_model_daily": {},
}
def _queue_and_build_initializing_city_payload(city: str, *, kind: str) -> Dict[str, Any]:
_enqueue_collector_refresh_request(city, kind, reason="cold_start")
return _build_initializing_city_payload(city, detail_depth=kind)
async def _get_or_start_city_force_refresh_task(
key: str,
refresh_factory: Callable[[], Awaitable[Dict[str, Any]]],
@@ -158,16 +285,37 @@ async def _refresh_city_payload_with_stale_timeout(
kind: str,
refresh_factory: Callable[[], Awaitable[Dict[str, Any]]],
) -> Dict[str, Any]:
cached_before_refresh = await _get_cached_city_payload(city, kind)
if not cached_before_refresh:
canonical_payload = await _get_canonical_city_payload(city, detail_depth=kind)
if canonical_payload:
_enqueue_collector_refresh_request(city, kind, reason="force_refresh")
logger.warning(
"city force refresh returning canonical latest without sync refresh city={} kind={}",
city,
kind,
)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind=kind)
task, started = await _get_or_start_city_force_refresh_task(f"{kind}:{city}", refresh_factory)
if not started:
cached_payload = await _get_cached_city_payload(city, kind)
if cached_payload:
if cached_before_refresh:
logger.warning(
"city force refresh already running city={} kind={}; returning stale cache",
city,
kind,
)
return await _overlay_cached_wunderground(city, cached_payload)
return await _overlay_cached_wunderground(city, cached_before_refresh)
canonical_payload = await _get_canonical_city_payload(city, detail_depth=kind)
if canonical_payload:
_enqueue_collector_refresh_request(city, kind, reason="force_refresh")
logger.warning(
"city force refresh returning canonical latest while refresh runs city={} kind={}",
city,
kind,
)
return canonical_payload
timeout_sec = _city_force_refresh_timeout_sec()
try:
return await asyncio.wait_for(asyncio.shield(task), timeout=timeout_sec)
@@ -384,9 +532,17 @@ async def _get_city_full_data(city: str, *, force_refresh: bool) -> Dict[str, An
if payload:
_start_city_full_stale_refresh(city)
return await _overlay_cached_wunderground(city, payload)
return await _refresh_city_full_data(city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="full")
if canonical_payload:
_request_city_full_refresh(city)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="full")
return await _overlay_cached_wunderground(city, payload)
return await _refresh_city_full_data(city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="full")
if canonical_payload:
_request_city_full_refresh(city)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="full")
async def _get_city_chart_data(city: str, *, force_refresh: bool) -> Dict[str, Any]:
@@ -714,9 +870,13 @@ async def get_city_detail_payload(
if payload:
_start_city_cache_stale_refresh(city, "panel", legacy_routes._refresh_city_panel_cache)
return await _overlay_cached_wunderground(city, payload)
return await run_in_threadpool(legacy_routes._refresh_city_panel_cache, city, False)
return _queue_and_build_initializing_city_payload(city, kind="panel")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
return await run_in_threadpool(legacy_routes._refresh_city_panel_cache, city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="panel")
if canonical_payload:
_request_city_cache_refresh(city, "panel", legacy_routes._refresh_city_panel_cache)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="panel")
if detail_mode == "nearby":
if force_refresh:
return await _refresh_city_cache_with_stale_timeout(
@@ -731,9 +891,17 @@ async def get_city_detail_payload(
if payload:
_start_city_cache_stale_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
return await _overlay_cached_wunderground(city, payload)
return await run_in_threadpool(legacy_routes._refresh_city_nearby_cache, city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="nearby")
if canonical_payload:
_request_city_cache_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="nearby")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
return await run_in_threadpool(legacy_routes._refresh_city_nearby_cache, city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="nearby")
if canonical_payload:
_request_city_cache_refresh(city, "nearby", legacy_routes._refresh_city_nearby_cache)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="nearby")
if detail_mode == "market":
if force_refresh:
return await _refresh_city_cache_with_stale_timeout(
@@ -748,9 +916,17 @@ async def get_city_detail_payload(
if payload:
_start_city_cache_stale_refresh(city, "market", legacy_routes._refresh_city_market_cache)
return await _overlay_cached_wunderground(city, payload)
return await run_in_threadpool(legacy_routes._refresh_city_market_cache, city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="market")
if canonical_payload:
_request_city_cache_refresh(city, "market", legacy_routes._refresh_city_market_cache)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="market")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
return await run_in_threadpool(legacy_routes._refresh_city_market_cache, city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="market")
if canonical_payload:
_request_city_cache_refresh(city, "market", legacy_routes._refresh_city_market_cache)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="market")
return await run_in_threadpool(legacy_routes._analyze, city, force_refresh, False, detail_mode)
@@ -774,9 +950,13 @@ async def get_city_summary_payload(
if payload:
_start_city_cache_stale_refresh(city, "summary", legacy_routes._refresh_city_summary_cache)
return await _overlay_cached_wunderground(city, payload)
return await run_in_threadpool(legacy_routes._refresh_city_summary_cache, city, False)
return _queue_and_build_initializing_city_payload(city, kind="summary")
return await _overlay_cached_wunderground(city, cached_entry.get("payload") or {})
return await run_in_threadpool(legacy_routes._refresh_city_summary_cache, city, False)
canonical_payload = await _get_canonical_city_payload(city, detail_depth="summary")
if canonical_payload:
_request_city_cache_refresh(city, "summary", legacy_routes._refresh_city_summary_cache)
return canonical_payload
return _queue_and_build_initializing_city_payload(city, kind="summary")
async def get_city_detail_aggregate_payload(
@@ -803,6 +983,8 @@ async def get_city_detail_aggregate_payload(
"full_data",
lambda: _get_city_full_data(city, force_refresh=force_refresh),
)
if isinstance(data, dict) and data.get("status") == "initializing":
return data
return await timer.measure_async(
"detail_payload",
+62 -22
View File
@@ -1,9 +1,8 @@
"""Lightweight realtime temperature stream for scrolling chart.
Maintains per-city deque buffers (max 1440 points) fed by _analyze()
refreshes. The /api/city/{name}/realtime-stream endpoint reads from
these buffers and returns a simple {points, thresholds} payload that
the frontend RealtimeScrollChart polls every 30 seconds.
Maintains per-city deque buffers (max 1440 points) fed by cached latest
observations. The polling endpoint must not trigger external weather
fetches; collectors and cache refreshers feed DB state.
"""
from __future__ import annotations
@@ -13,12 +12,14 @@ import threading
import time
from typing import Any, Dict, List, Optional
from web.analysis_service import _analyze
from src.database.db_manager import DBManager
# Per-city ring buffers: city_name → deque of {timestamp, temp, source}
_STREAM_BUFFERS: Dict[str, collections.deque] = {}
_BUFFER_LOCK = threading.Lock()
_MAXLEN = 1440
_CACHE_DB = DBManager()
_analyze = None # compatibility placeholder for older tests/monkeypatches
def _best_temp(data: Dict[str, Any]) -> Optional[float]:
@@ -76,19 +77,63 @@ def _extract_thresholds(data: Dict[str, Any]) -> List[Dict[str, Any]]:
return thresholds
def _cached_city_payload(city: str) -> Dict[str, Any]:
normalized_city = str(city or "").strip().lower()
if not normalized_city:
return {}
for kind in ("panel", "full"):
try:
entry = _CACHE_DB.get_city_cache(kind, normalized_city)
except Exception:
continue
if not isinstance(entry, dict):
continue
payload = entry.get("payload") or {}
if isinstance(payload, dict) and payload:
return payload
return {}
def _latest_canonical_point(city: str) -> Optional[Dict[str, Any]]:
normalized_city = str(city or "").strip().lower()
if not normalized_city:
return None
try:
row = _CACHE_DB.get_canonical_temperature(normalized_city)
except Exception:
return None
if not isinstance(row, dict):
return None
canonical = row.get("payload") or row
if not isinstance(canonical, dict):
return None
try:
temp = float(canonical.get("value"))
except (TypeError, ValueError):
return None
timestamp = str(
canonical.get("observed_at")
or canonical.get("observed_at_local")
or canonical.get("fetched_at")
or time.strftime("%H:%M:%S")
)
source = str(canonical.get("source") or "canonical")
return {"timestamp": timestamp, "temp": round(temp, 1), "source": source}
def capture_sample(city: str) -> None:
"""Record one sample for *city* into its ring buffer."""
try:
data = _analyze(city, force_refresh=False, detail_mode="panel")
except Exception:
return
temp = _best_temp(data)
if temp is None:
return
ts = time.strftime("%H:%M:%S")
point = {"timestamp": ts, "temp": round(temp, 1), "source": "metar"}
point = _latest_canonical_point(city)
if point is None:
data = _cached_city_payload(city)
temp = _best_temp(data)
if temp is None:
return
current = data.get("current") if isinstance(data, dict) else {}
source = "cache"
if isinstance(current, dict):
source = str(current.get("source_code") or current.get("settlement_source") or source)
point = {"timestamp": time.strftime("%H:%M:%S"), "temp": round(temp, 1), "source": source}
with _BUFFER_LOCK:
buf = _STREAM_BUFFERS.get(city)
@@ -107,11 +152,6 @@ def get_realtime_stream_payload(city: str) -> Dict[str, Any]:
buf = _STREAM_BUFFERS.get(city)
points = list(buf) if buf else []
# Build thresholds from cached analysis
try:
data = _analyze(city, force_refresh=False, detail_mode="panel")
thresholds = _extract_thresholds(data)
except Exception:
thresholds = []
thresholds = _extract_thresholds(_cached_city_payload(city))
return {"points": points, "thresholds": thresholds}
+22
View File
@@ -30,6 +30,10 @@ from web.analysis_service import (
_build_city_market_scan_payload,
_build_city_summary_payload,
)
from web.services.canonical_temperature import (
attach_canonical_temperature,
store_canonical_temperature_from_payload,
)
from web.scan_terminal_service import build_scan_terminal_payload # noqa: F401 - compatibility export for tests and transitional routers
from web.core import (
CITIES,
@@ -275,9 +279,23 @@ def _refresh_market_scan_payload_from_cached_analysis(
return payload.get("market_scan_payload") or {}
def _attach_and_store_canonical_temperature(city: str, payload: dict) -> dict:
if not isinstance(payload, dict):
return payload
attach_canonical_temperature(payload, city=city)
try:
store_canonical_temperature_from_payload(_CACHE_DB, city, payload)
except Exception as exc:
logger.debug("canonical temperature store skipped city={}: {}", city, exc)
return payload
def _refresh_city_summary_cache(city: str, force_refresh: bool = False) -> dict:
data = _analyze_summary(city, force_refresh=force_refresh)
_attach_and_store_canonical_temperature(city, data)
payload = _build_city_summary_payload(data)
if data.get("canonical_temperature"):
payload["canonical_temperature"] = data["canonical_temperature"]
_CACHE_DB.set_city_cache(
"summary",
city,
@@ -290,6 +308,7 @@ def _refresh_city_summary_cache(city: str, force_refresh: bool = False) -> dict:
def _refresh_city_panel_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, detail_mode="panel")
_attach_and_store_canonical_temperature(city, payload)
_CACHE_DB.set_city_cache(
"panel",
city,
@@ -302,6 +321,7 @@ def _refresh_city_panel_cache(city: str, force_refresh: bool = False) -> dict:
def _refresh_city_nearby_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, detail_mode="nearby")
_attach_and_store_canonical_temperature(city, payload)
_CACHE_DB.set_city_cache(
"nearby",
city,
@@ -314,6 +334,7 @@ def _refresh_city_nearby_cache(city: str, force_refresh: bool = False) -> dict:
def _refresh_city_market_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, detail_mode="market")
_attach_and_store_canonical_temperature(city, payload)
now_ts = time.time()
payload["market_analysis_cached_at"] = datetime.now().isoformat()
payload["market_analysis_cached_at_ts"] = now_ts
@@ -330,6 +351,7 @@ def _refresh_city_market_cache(city: str, force_refresh: bool = False) -> dict:
def _refresh_city_full_cache(city: str, force_refresh: bool = False) -> dict:
payload = _analyze(city, force_refresh=force_refresh, detail_mode="full")
_attach_and_store_canonical_temperature(city, payload)
_CACHE_DB.set_city_cache(
"full",
city,
+6 -9
View File
@@ -10,7 +10,7 @@ from fastapi.concurrency import run_in_threadpool
from loguru import logger
import web.routes as legacy_routes
from web.services.city_api import _build_cities_payload
from web.services.city_api import _build_cities_payload, get_city_detail_payload
def _resolve_default_city(request: Request) -> Optional[str]:
@@ -37,15 +37,12 @@ async def build_dashboard_init_payload(request: Request) -> Dict[str, Any]:
detail_payload: Optional[Dict[str, Any]] = None
if default_city:
try:
cached_entry = await run_in_threadpool(
legacy_routes._CACHE_DB.get_city_cache, "panel", default_city,
detail_payload = await get_city_detail_payload(
request,
default_city,
force_refresh=False,
depth="panel",
)
if cached_entry:
detail_payload = cached_entry.get("payload") or {}
else:
detail_payload = await run_in_threadpool(
legacy_routes._refresh_city_panel_cache, default_city, False,
)
except Exception as exc:
logger.warning(
"dashboard_init default_city={} panel failed: {}", default_city, exc
+14 -7
View File
@@ -92,20 +92,27 @@ def run_system_priority_warm(
primary = list(batches.get("primary") or [])
secondary = list(batches.get("secondary") or [])
def _queue_city_refresh(city: str, *, priority: str) -> None:
enqueue = getattr(legacy_routes._CACHE_DB, "enqueue_observation_refresh_request", None)
if not callable(enqueue):
logger.warning("priority warm queue unavailable city={} timezone={}", city, timezone)
return
enqueue(
city=city,
kind="panel",
priority=priority,
reason="system_priority_warm",
)
def _runner() -> None:
for city in primary:
try:
legacy_routes._refresh_city_summary_cache(city, force_refresh=False)
legacy_routes._refresh_city_panel_cache(city, force_refresh=False)
legacy_routes._refresh_city_nearby_cache(city, force_refresh=False)
legacy_routes._refresh_city_market_cache(city, force_refresh=False)
legacy_routes._refresh_city_full_cache(city, force_refresh=False)
_queue_city_refresh(city, priority="high")
except Exception as exc:
logger.warning("priority warm primary failed city={} timezone={}: {}", city, timezone, exc)
for city in secondary:
try:
legacy_routes._refresh_city_summary_cache(city, force_refresh=False)
legacy_routes._refresh_city_panel_cache(city, force_refresh=False)
_queue_city_refresh(city, priority="normal")
except Exception as exc:
logger.warning("priority warm secondary failed city={} timezone={}: {}", city, timezone, exc)