Decouple live weather reads from source collection

This commit is contained in:
2569718930@qq.com
2026-06-14 17:53:24 +08:00
parent a098a5ab20
commit e111031119
18 changed files with 2248 additions and 117 deletions
+541
View File
@@ -524,6 +524,111 @@ class DBManager:
source_fingerprint TEXT
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS canonical_temperature_latest (
city TEXT PRIMARY KEY,
payload_json TEXT NOT NULL,
value REAL,
source TEXT,
source_role TEXT,
observed_at TEXT,
fetched_at TEXT,
freshness_sec INTEGER,
freshness_status TEXT,
confidence REAL,
explanation TEXT,
updated_at TEXT NOT NULL,
updated_at_ts REAL NOT NULL
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_canonical_temperature_latest_updated
ON canonical_temperature_latest(updated_at_ts DESC)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS raw_observation_store (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL,
city TEXT NOT NULL,
station_code TEXT NOT NULL DEFAULT '',
station_name TEXT NOT NULL DEFAULT '',
runway TEXT NOT NULL DEFAULT '',
value REAL,
value_unit TEXT NOT NULL DEFAULT '',
observed_at TEXT,
fetched_at TEXT NOT NULL,
source_latency_sec REAL,
status TEXT NOT NULL DEFAULT 'ok',
error_count INTEGER NOT NULL DEFAULT 0,
last_success_at TEXT,
payload_json TEXT NOT NULL,
created_at_ts REAL NOT NULL
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_raw_observation_store_source_city_time
ON raw_observation_store(source, city, observed_at DESC, fetched_at DESC)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS raw_observation_latest (
source TEXT NOT NULL,
city TEXT NOT NULL,
station_code TEXT NOT NULL DEFAULT '',
station_name TEXT NOT NULL DEFAULT '',
runway TEXT NOT NULL DEFAULT '',
value REAL,
value_unit TEXT NOT NULL DEFAULT '',
observed_at TEXT,
fetched_at TEXT NOT NULL,
source_latency_sec REAL,
status TEXT NOT NULL DEFAULT 'ok',
error_count INTEGER NOT NULL DEFAULT 0,
last_success_at TEXT,
payload_json TEXT NOT NULL,
updated_at_ts REAL NOT NULL,
PRIMARY KEY (source, city, station_code, runway)
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_raw_observation_latest_city_source
ON raw_observation_latest(city, source, updated_at_ts DESC)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS observation_refresh_requests (
id INTEGER PRIMARY KEY AUTOINCREMENT,
city TEXT NOT NULL,
kind TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT '',
priority TEXT NOT NULL DEFAULT 'normal',
reason TEXT NOT NULL DEFAULT '',
status TEXT NOT NULL DEFAULT 'pending',
attempts INTEGER NOT NULL DEFAULT 0,
owner TEXT NOT NULL DEFAULT '',
requested_at TEXT NOT NULL,
requested_at_ts REAL NOT NULL,
claimed_at_ts REAL,
completed_at_ts REAL,
last_error TEXT NOT NULL DEFAULT ''
)
""")
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_observation_refresh_requests_status
ON observation_refresh_requests(status, priority, requested_at_ts)
"""
)
conn.execute(
"""
CREATE INDEX IF NOT EXISTS idx_observation_refresh_requests_city
ON observation_refresh_requests(city, kind, status)
"""
)
conn.execute("""
CREATE TABLE IF NOT EXISTS cache_refresh_locks (
cache_key TEXT PRIMARY KEY,
@@ -727,6 +832,24 @@ class DBManager:
conn.commit()
logger.info(f"Database initialized successfully path={self.db_path}")
@staticmethod
def _float_or_none(value: Any) -> Optional[float]:
try:
if value is None or value == "":
return None
return float(value)
except (TypeError, ValueError):
return None
@staticmethod
def _int_or_none(value: Any) -> Optional[int]:
try:
if value is None or value == "":
return None
return int(float(value))
except (TypeError, ValueError):
return None
def _cache_table_name(self, kind: str) -> Optional[str]:
normalized = str(kind or "").strip().lower()
if normalized == "summary":
@@ -819,6 +942,424 @@ class DBManager:
)
conn.commit()
def get_canonical_temperature(self, city: str) -> Optional[Dict[str, Any]]:
normalized_city = str(city or "").strip().lower()
if not normalized_city:
return None
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
row = conn.execute(
"""
SELECT city, payload_json, value, source, source_role, observed_at,
fetched_at, freshness_sec, freshness_status, confidence,
explanation, updated_at, updated_at_ts
FROM canonical_temperature_latest
WHERE city = ?
LIMIT 1
""",
(normalized_city,),
).fetchone()
if not row:
return None
try:
payload = json.loads(str(row["payload_json"] or "{}"))
except Exception:
return None
if not isinstance(payload, dict):
return None
return {
"city": str(row["city"] or normalized_city),
"payload": payload,
"value": self._float_or_none(row["value"]),
"source": str(row["source"] or ""),
"source_role": str(row["source_role"] or ""),
"observed_at": str(row["observed_at"] or ""),
"fetched_at": str(row["fetched_at"] or ""),
"freshness_sec": self._int_or_none(row["freshness_sec"]),
"freshness_status": str(row["freshness_status"] or ""),
"confidence": self._float_or_none(row["confidence"]),
"explanation": str(row["explanation"] or ""),
"updated_at": str(row["updated_at"] or ""),
"updated_at_ts": float(row["updated_at_ts"] or 0.0),
}
def set_canonical_temperature(self, city: str, payload: Dict[str, Any]) -> None:
normalized_city = str(city or "").strip().lower()
if not normalized_city or not isinstance(payload, dict):
return
value = self._float_or_none(payload.get("value"))
source = str(payload.get("source") or "").strip().lower()
source_role = str(payload.get("source_role") or "").strip().lower()
observed_at = str(payload.get("observed_at") or "").strip()
fetched_at = str(payload.get("fetched_at") or "").strip()
freshness_sec = self._int_or_none(payload.get("freshness_sec"))
freshness_status = str(payload.get("freshness_status") or "").strip().lower()
confidence = self._float_or_none(payload.get("confidence"))
explanation = str(payload.get("explanation") or "").strip()
now_dt = datetime.now()
now = now_dt.isoformat()
now_ts = now_dt.timestamp()
with self._get_connection() as conn:
conn.execute(
"""
INSERT INTO canonical_temperature_latest (
city, payload_json, value, source, source_role, observed_at,
fetched_at, freshness_sec, freshness_status, confidence,
explanation, updated_at, updated_at_ts
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(city) DO UPDATE SET
payload_json = excluded.payload_json,
value = excluded.value,
source = excluded.source,
source_role = excluded.source_role,
observed_at = excluded.observed_at,
fetched_at = excluded.fetched_at,
freshness_sec = excluded.freshness_sec,
freshness_status = excluded.freshness_status,
confidence = excluded.confidence,
explanation = excluded.explanation,
updated_at = excluded.updated_at,
updated_at_ts = excluded.updated_at_ts
""",
(
normalized_city,
json.dumps({**payload, "city": normalized_city}, ensure_ascii=False),
value,
source,
source_role,
observed_at,
fetched_at,
freshness_sec,
freshness_status,
confidence,
explanation,
now,
now_ts,
),
)
conn.commit()
def append_raw_observation(
self,
*,
source: str,
city: str,
value: Any = None,
observed_at: str = "",
fetched_at: str = "",
station_code: str = "",
station_name: str = "",
runway: str = "",
value_unit: str = "",
source_latency_sec: Any = None,
status: str = "ok",
error_count: int = 0,
last_success_at: str = "",
payload: Optional[Dict[str, Any]] = None,
) -> None:
normalized_source = str(source or "").strip().lower()
normalized_city = str(city or "").strip().lower()
if not normalized_source or not normalized_city:
return
safe_station_code = str(station_code or "").strip().upper()
safe_station_name = str(station_name or "").strip()
safe_runway = str(runway or "").strip().upper()
safe_observed_at = str(observed_at or "").strip()
now_dt = datetime.now()
safe_fetched_at = str(fetched_at or now_dt.isoformat()).strip()
safe_status = str(status or "ok").strip().lower() or "ok"
value_float = self._float_or_none(value)
latency_float = self._float_or_none(source_latency_sec)
payload_json = json.dumps(payload or {}, ensure_ascii=False)
created_at_ts = now_dt.timestamp()
success_at = str(last_success_at or (safe_fetched_at if safe_status == "ok" else "")).strip()
with self._get_connection() as conn:
conn.execute(
"""
INSERT INTO raw_observation_store (
source, city, station_code, station_name, runway, value,
value_unit, observed_at, fetched_at, source_latency_sec,
status, error_count, last_success_at, payload_json, created_at_ts
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
normalized_source,
normalized_city,
safe_station_code,
safe_station_name,
safe_runway,
value_float,
str(value_unit or "").strip(),
safe_observed_at,
safe_fetched_at,
latency_float,
safe_status,
max(0, int(error_count or 0)),
success_at,
payload_json,
created_at_ts,
),
)
conn.execute(
"""
INSERT INTO raw_observation_latest (
source, city, station_code, station_name, runway, value,
value_unit, observed_at, fetched_at, source_latency_sec,
status, error_count, last_success_at, payload_json, updated_at_ts
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source, city, station_code, runway) DO UPDATE SET
station_name = excluded.station_name,
value = excluded.value,
value_unit = excluded.value_unit,
observed_at = excluded.observed_at,
fetched_at = excluded.fetched_at,
source_latency_sec = excluded.source_latency_sec,
status = excluded.status,
error_count = excluded.error_count,
last_success_at = excluded.last_success_at,
payload_json = excluded.payload_json,
updated_at_ts = excluded.updated_at_ts
""",
(
normalized_source,
normalized_city,
safe_station_code,
safe_station_name,
safe_runway,
value_float,
str(value_unit or "").strip(),
safe_observed_at,
safe_fetched_at,
latency_float,
safe_status,
max(0, int(error_count or 0)),
success_at,
payload_json,
created_at_ts,
),
)
conn.commit()
def get_latest_raw_observation(
self,
source: str,
city: str,
*,
station_code: str = "",
runway: str = "",
) -> Optional[Dict[str, Any]]:
normalized_source = str(source or "").strip().lower()
normalized_city = str(city or "").strip().lower()
if not normalized_source or not normalized_city:
return None
filters = ["source = ?", "city = ?"]
params: List[Any] = [normalized_source, normalized_city]
if station_code:
filters.append("station_code = ?")
params.append(str(station_code or "").strip().upper())
if runway:
filters.append("runway = ?")
params.append(str(runway or "").strip().upper())
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
row = conn.execute(
f"""
SELECT *
FROM raw_observation_latest
WHERE {' AND '.join(filters)}
ORDER BY updated_at_ts DESC
LIMIT 1
""",
params,
).fetchone()
if not row:
return None
try:
payload = json.loads(str(row["payload_json"] or "{}"))
except Exception:
payload = {}
if not isinstance(payload, dict):
payload = {}
return {
"source": str(row["source"] or ""),
"city": str(row["city"] or ""),
"station_code": str(row["station_code"] or ""),
"station_name": str(row["station_name"] or ""),
"runway": str(row["runway"] or ""),
"value": self._float_or_none(row["value"]),
"value_unit": str(row["value_unit"] or ""),
"observed_at": str(row["observed_at"] or ""),
"fetched_at": str(row["fetched_at"] or ""),
"source_latency_sec": self._float_or_none(row["source_latency_sec"]),
"status": str(row["status"] or ""),
"error_count": int(row["error_count"] or 0),
"last_success_at": str(row["last_success_at"] or ""),
"payload": payload,
"updated_at_ts": float(row["updated_at_ts"] or 0.0),
}
def enqueue_observation_refresh_request(
self,
*,
city: str,
kind: str = "",
source: str = "",
priority: str = "normal",
reason: str = "",
) -> bool:
normalized_city = str(city or "").strip().lower()
normalized_kind = str(kind or "").strip().lower()
normalized_source = str(source or "").strip().lower()
normalized_priority = str(priority or "normal").strip().lower()
if normalized_priority not in {"high", "normal", "low"}:
normalized_priority = "normal"
if not normalized_city:
return False
priority_rank = {"low": 0, "normal": 1, "high": 2}
now_dt = datetime.now()
now = now_dt.isoformat()
now_ts = now_dt.timestamp()
with self._get_connection() as conn:
existing = conn.execute(
"""
SELECT id, priority
FROM observation_refresh_requests
WHERE city = ? AND source = ? AND status IN ('pending', 'claimed')
ORDER BY requested_at_ts DESC
LIMIT 1
""",
(normalized_city, normalized_source),
).fetchone()
if existing:
existing_priority = str(existing[1] or "normal").strip().lower()
if priority_rank.get(existing_priority, 1) > priority_rank[normalized_priority]:
normalized_priority = existing_priority
conn.execute(
"""
UPDATE observation_refresh_requests
SET kind = ?, priority = ?, reason = ?, requested_at = ?, requested_at_ts = ?
WHERE id = ?
""",
(
normalized_kind,
normalized_priority,
str(reason or "").strip(),
now,
now_ts,
int(existing[0]),
),
)
else:
conn.execute(
"""
INSERT INTO observation_refresh_requests (
city, kind, source, priority, reason, status,
requested_at, requested_at_ts
)
VALUES (?, ?, ?, ?, ?, 'pending', ?, ?)
""",
(
normalized_city,
normalized_kind,
normalized_source,
normalized_priority,
str(reason or "").strip(),
now,
now_ts,
),
)
conn.commit()
return True
def claim_observation_refresh_requests(
self,
*,
limit: int = 20,
owner: str = "",
now_ts: Optional[float] = None,
) -> List[Dict[str, Any]]:
safe_limit = max(1, min(int(limit or 20), 200))
safe_owner = str(owner or "").strip() or secrets.token_hex(6)
claim_ts = float(now_ts if now_ts is not None else datetime.now().timestamp())
with self._get_connection() as conn:
conn.row_factory = sqlite3.Row
rows = conn.execute(
"""
SELECT *
FROM observation_refresh_requests
WHERE status = 'pending'
ORDER BY
CASE priority WHEN 'high' THEN 0 WHEN 'normal' THEN 1 ELSE 2 END,
requested_at_ts ASC
LIMIT ?
""",
(safe_limit,),
).fetchall()
ids = [int(row["id"]) for row in rows]
if ids:
placeholders = ",".join("?" for _ in ids)
conn.execute(
f"""
UPDATE observation_refresh_requests
SET status = 'claimed',
owner = ?,
attempts = attempts + 1,
claimed_at_ts = ?
WHERE id IN ({placeholders})
""",
[safe_owner, claim_ts, *ids],
)
conn.commit()
return [
{
"id": int(row["id"]),
"city": str(row["city"] or ""),
"kind": str(row["kind"] or ""),
"source": str(row["source"] or ""),
"priority": str(row["priority"] or ""),
"reason": str(row["reason"] or ""),
"status": "claimed",
"attempts": int(row["attempts"] or 0) + 1,
"owner": safe_owner,
"requested_at": str(row["requested_at"] or ""),
"requested_at_ts": float(row["requested_at_ts"] or 0.0),
"claimed_at_ts": claim_ts,
"last_error": str(row["last_error"] or ""),
}
for row in rows
]
def mark_observation_refresh_request_done(
self,
request_id: int,
*,
status: str = "done",
error: str = "",
) -> None:
safe_status = str(status or "done").strip().lower()
if safe_status not in {"done", "failed"}:
safe_status = "done"
with self._get_connection() as conn:
conn.execute(
"""
UPDATE observation_refresh_requests
SET status = ?,
completed_at_ts = ?,
last_error = ?
WHERE id = ?
""",
(
safe_status,
datetime.now().timestamp(),
str(error or "").strip(),
int(request_id),
),
)
conn.commit()
def acquire_cache_refresh_lock(
self,
cache_key: str,
+28 -7
View File
@@ -27,6 +27,7 @@ from src.utils.telegram_i18n import (
normalize_push_language as _normalize_push_language,
telegram_push_language as _resolve_telegram_push_language,
)
from web.services.canonical_temperature import build_city_weather_from_canonical
# Forum topic routing: maps city_key -> message_thread_id for the push forum group.
# Created by scripts/create_forum_topics.py, stored in the runtime data dir.
@@ -1504,9 +1505,11 @@ def _cached_payload_observation_epoch(payload: Dict[str, Any]) -> Optional[int]:
airport_current = payload.get("airport_current") or {}
current = payload.get("current") or {}
candidates = [
(payload.get("canonical_temperature") or {}).get("observed_at"),
amos.get("observation_time"),
airport_primary.get("obs_time"),
airport_current.get("obs_time"),
current.get("observed_at"),
current.get("obs_time"),
]
parsed = [
@@ -1574,19 +1577,37 @@ def _read_cached_airport_city_weather(city: str, max_age_sec: Optional[int] = No
return None
def _read_canonical_airport_city_weather(city: str) -> Optional[Dict[str, Any]]:
normalized_city = (city or "").strip().lower()
if not normalized_city:
return None
try:
db = DBManager()
getter = getattr(db, "get_canonical_temperature", None)
if not callable(getter):
return None
row = getter(normalized_city)
except Exception as exc:
logger.debug("airport push canonical latest read failed city={}: {}", normalized_city, exc)
return None
if not isinstance(row, dict):
return None
canonical = row.get("payload") or row
if not isinstance(canonical, dict):
return None
return build_city_weather_from_canonical(normalized_city, canonical)
def _load_airport_city_weather_for_push(city: str) -> Dict[str, Any]:
cached = _read_cached_airport_city_weather(city)
if cached is not None:
return cached
from web.app import _analyze # lazy import - only the bot process needs it
canonical = _read_canonical_airport_city_weather(city)
if canonical is not None:
return canonical
return _analyze(
city,
force_refresh=False,
force_refresh_observations_only=False,
detail_mode="panel",
)
raise RuntimeError(f"no cached city weather for airport push city={city}")
# Per-city temperature window threshold (°C below DEB predicted high)