gavindiaz/PolyWeather
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restore DEB training settlement worker

Browse Source
This commit is contained in:
2569718930@qq.com
2026-06-23 18:16:46 +08:00
parent 0c76874418
commit ba2db42eaa
10 changed files with 700 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/diagnostics/health.py
+37 -1
View File
@@ -133,15 +133,28 @@ def _table_date_summary(conn, table_name: str) -> Dict[str, Any]:
except Exception as exc:
return {"ok": False, "error": str(exc), "row_count": 0, "cities_count": 0}
max_date = row["max_date"]
return {
"ok": True,
"row_count": int(row["row_count"] or 0),
"cities_count": int(row["cities_count"] or 0),
"min_date": row["min_date"],
"max_date": row["max_date"],
"max_date": max_date,
"stale_days": _target_date_stale_days(max_date),
}
def _target_date_stale_days(target_date: Optional[str]) -> Optional[int]:
raw = str(target_date or "").strip()
if not raw:
return None
try:
parsed = datetime.strptime(raw, "%Y-%m-%d").date()
except Exception:
return None
return max(0, (datetime.now(timezone.utc).date() - parsed).days)
def _truth_source_counts(conn) -> Dict[str, int]:
try:
rows = conn.execute(
@@ -293,11 +306,17 @@ def _training_data_summary(account_db, city_registry) -> Dict[str, Any]:
import sqlite3
db_path = account_db.db_path
daily_records = {"ok": False, "row_count": 0, "cities_count": 0}
truth_records = {"ok": False, "row_count": 0, "cities_count": 0}
truth_revisions = {"ok": False, "row_count": 0}
training_features = {"ok": False, "row_count": 0, "cities_count": 0}
stale_threshold_days = max(
0,
int(os.getenv("POLYWEATHER_TRAINING_DATA_STALE_WARN_DAYS", "2") or "2"),
)
try:
with connect_sqlite(db_path, row_factory=sqlite3.Row) as conn:
daily_records = _table_date_summary(conn, "daily_records_store")
truth_records = _table_date_summary(conn, "truth_records_store")
if truth_records.get("ok"):
truth_records["source_counts"] = _truth_source_counts(conn)
@@ -311,19 +330,36 @@ def _training_data_summary(account_db, city_registry) -> Dict[str, Any]:
"db_path": db_path,
"db_ok": False,
"error": str(exc),
"daily_records": daily_records,
"truth_records": truth_records,
"truth_revisions": truth_revisions,
"training_features": training_features,
"stale": True,
"stale_threshold_days": stale_threshold_days,
"city_coverage": {},
"model_city_coverage": {},
}
stale_days = [
value
for value in (
daily_records.get("stale_days"),
truth_records.get("stale_days"),
training_features.get("stale_days"),
)
if value is not None
]
return {
"db_path": db_path,
"db_ok": True,
"daily_records": daily_records,
"truth_records": truth_records,
"truth_revisions": truth_revisions,
"training_features": training_features,
"stale": any(int(value) > stale_threshold_days for value in stale_days)
if stale_days
else True,
"stale_threshold_days": stale_threshold_days,
"city_coverage": city_coverage,
"model_city_coverage": _model_city_coverage_summary(
city_coverage.get("entries") or [],
web/services/system_api.py
+50
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import time
import os
from collections import Counter
from datetime import datetime, timezone
from typing import Any, Dict, Optional
from fastapi import BackgroundTasks, Request
@@ -13,6 +14,7 @@ from fastapi.responses import PlainTextResponse
from loguru import logger
from src.database.db_manager import DBManager
from src.database.sqlite_connection import connect_sqlite
from src.utils.metrics import export_prometheus_metrics, gauge_set
from web.core import build_health_payload, build_system_status_payload
import web.routes as legacy_routes
@@ -211,3 +213,51 @@ def _refresh_operational_metrics() -> None:
gauge_set("polyweather_sqlite_db_size_bytes", os.path.getsize(db_path))
except OSError:
pass
_refresh_training_data_metrics(db_path)
def _target_date_stale_days(target_date: Optional[str]) -> Optional[int]:
raw = str(target_date or "").strip()
if not raw:
return None
try:
parsed = datetime.strptime(raw, "%Y-%m-%d").date()
except Exception:
return None
return max(0, (datetime.now(timezone.utc).date() - parsed).days)
def _max_target_date(db_path: str, table_name: str) -> Optional[str]:
with connect_sqlite(db_path) as conn:
row = conn.execute(f"SELECT MAX(target_date) AS max_date FROM {table_name}").fetchone()
if not row:
return None
return row[0]
def _refresh_training_data_metrics(db_path: str) -> None:
stale_threshold_days = max(
0,
int(os.getenv("POLYWEATHER_TRAINING_DATA_STALE_WARN_DAYS", "2") or "2"),
)
table_metrics = {
"daily_records_store": "polyweather_daily_records_stale_days",
"truth_records_store": "polyweather_truth_records_stale_days",
"training_feature_records_store": "polyweather_training_features_stale_days",
}
max_stale_days: Optional[int] = None
for table_name, metric_name in table_metrics.items():
try:
stale_days = _target_date_stale_days(_max_target_date(db_path, table_name))
except Exception:
stale_days = None
if stale_days is None:
gauge_set(metric_name, -1)
max_stale_days = max(max_stale_days or 0, stale_threshold_days + 1)
continue
gauge_set(metric_name, stale_days)
max_stale_days = max(max_stale_days or 0, stale_days)
gauge_set(
"polyweather_training_data_stale",
1 if max_stale_days is None or max_stale_days > stale_threshold_days else 0,
)
web/training_settlement_service.py
+142
View File
@@ -0,0 +1,142 @@
"""Low-frequency DEB training settlement maintenance."""
from __future__ import annotations
from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Sequence
from loguru import logger
from src.analysis.deb_algorithm import reconcile_recent_actual_highs
from src.data_collection.city_registry import CITY_REGISTRY
AnalysisRunner = Callable[[str], Mapping[str, Any]]
ActualReconciler = Callable[..., Mapping[str, Any]]
UNSUPPORTED_SETTLEMENT_SOURCES = {"wunderground"}
RECONCILE_SETTLEMENT_SOURCES = {"metar", "hko", "noaa"}
def _normalize_city(city: str) -> str:
return str(city or "").strip().lower().replace("-", " ")
def _selected_city_names(
city_registry: Mapping[str, Mapping[str, Any]],
cities: Optional[Iterable[str]],
) -> Sequence[str]:
selected = {_normalize_city(city) for city in (cities or []) if _normalize_city(city)}
names = []
for city in sorted(city_registry.keys()):
normalized = _normalize_city(city)
if selected and normalized not in selected:
continue
names.append(normalized)
return tuple(names)
def _is_supported_training_city(city_meta: Mapping[str, Any]) -> bool:
source = str(city_meta.get("settlement_source") or "metar").strip().lower()
if source in UNSUPPORTED_SETTLEMENT_SOURCES:
return False
return bool(
str(city_meta.get("icao") or "").strip()
or str(city_meta.get("settlement_station_code") or "").strip()
or source in {"hko", "cwa", "ims", "ncm", "aeroweb"}
)
def _can_reconcile_actual_history(city_meta: Mapping[str, Any]) -> bool:
source = str(city_meta.get("settlement_source") or "metar").strip().lower()
return source in RECONCILE_SETTLEMENT_SOURCES
def _default_analysis_runner(city: str) -> Mapping[str, Any]:
from web.analysis_service import _analyze
return _analyze(city, force_refresh=False, detail_mode="panel")
def _default_actual_reconciler(city: str, *, lookback_days: int) -> Mapping[str, Any]:
return reconcile_recent_actual_highs(city, lookback_days=lookback_days)
def run_training_settlement_cycle(
*,
city_registry: Optional[Mapping[str, Mapping[str, Any]]] = None,
cities: Optional[Iterable[str]] = None,
analysis_runner: Optional[AnalysisRunner] = None,
actual_reconciler: Optional[ActualReconciler] = None,
lookback_days: int = 10,
) -> Dict[str, Any]:
registry = city_registry or CITY_REGISTRY
run_analysis = analysis_runner or _default_analysis_runner
reconcile_actual = actual_reconciler or _default_actual_reconciler
safe_lookback = max(1, int(lookback_days or 1))
processed = 0
failed = 0
unsupported = 0
items = []
for city in _selected_city_names(registry, cities):
meta = registry.get(city) or {}
if not _is_supported_training_city(meta):
unsupported += 1
items.append(
{
"city": city,
"ok": True,
"status": "unsupported",
"reason": "unsupported_settlement_source",
}
)
continue
try:
analysis_payload = run_analysis(city)
if _can_reconcile_actual_history(meta):
reconcile_payload = reconcile_actual(city, lookback_days=safe_lookback)
else:
reconcile_payload = {
"ok": True,
"updated": 0,
"reason": "unsupported_reconcile_source",
"source": str(meta.get("settlement_source") or "").strip().lower(),
}
reconcile_ok = bool(reconcile_payload.get("ok", True))
if reconcile_ok:
processed += 1
else:
failed += 1
items.append(
{
"city": city,
"ok": reconcile_ok,
"status": "processed" if reconcile_ok else "failed",
"analysis_status": str(
(analysis_payload or {}).get("status") or "ok"
),
"reconcile": dict(reconcile_payload or {}),
}
)
except Exception as exc:
failed += 1
logger.warning("training settlement city failed city={}: {}", city, exc)
items.append(
{
"city": city,
"ok": False,
"status": "failed",
"error": str(exc),
}
)
return {
"ok": failed == 0,
"processed": processed,
"failed": failed,
"unsupported": unsupported,
"lookback_days": safe_lookback,
"items": items,
}
web/training_settlement_worker.py
+103
View File
@@ -0,0 +1,103 @@
"""Standalone low-frequency DEB training settlement worker."""
from __future__ import annotations
import argparse
import json
import os
import signal
import threading
import time
from types import FrameType
from typing import Optional
from loguru import logger
from web.training_settlement_service import run_training_settlement_cycle
_STOP_EVENT = threading.Event()
def _env_int(name: str, default: int) -> int:
raw = os.getenv(name)
if raw is None:
return default
try:
return int(raw)
except Exception:
return default
def _handle_stop_signal(signum: int, _frame: Optional[FrameType]) -> None:
logger.info("training settlement worker stopping signal={}", signum)
_STOP_EVENT.set()
def _parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Run low-frequency DEB training settlement maintenance."
)
parser.add_argument("--once", action="store_true", help="Run one cycle and exit.")
parser.add_argument(
"--interval-sec",
type=int,
default=_env_int("POLYWEATHER_TRAINING_SETTLEMENT_INTERVAL_SEC", 21600),
)
parser.add_argument(
"--initial-delay-sec",
type=int,
default=_env_int("POLYWEATHER_TRAINING_SETTLEMENT_INITIAL_DELAY_SEC", 60),
)
parser.add_argument(
"--lookback-days",
type=int,
default=_env_int("POLYWEATHER_TRAINING_SETTLEMENT_LOOKBACK_DAYS", 10),
)
parser.add_argument("--cities", nargs="*", default=None)
return parser.parse_args()
def _run_once(*, lookback_days: int, cities: Optional[list[str]]) -> dict:
result = run_training_settlement_cycle(
cities=cities,
lookback_days=lookback_days,
)
logger.info("training settlement result={}", json.dumps(result, ensure_ascii=False))
return result
def main() -> None:
signal.signal(signal.SIGINT, _handle_stop_signal)
signal.signal(signal.SIGTERM, _handle_stop_signal)
args = _parse_args()
if args.once:
result = _run_once(lookback_days=args.lookback_days, cities=args.cities)
print(json.dumps(result, ensure_ascii=False))
return
interval_sec = max(300, int(args.interval_sec or 21600))
initial_delay_sec = max(0, int(args.initial_delay_sec or 0))
logger.info(
"training settlement worker started interval={}s lookback_days={}",
interval_sec,
args.lookback_days,
)
if initial_delay_sec and _STOP_EVENT.wait(initial_delay_sec):
return
while not _STOP_EVENT.is_set():
started = time.time()
try:
_run_once(lookback_days=args.lookback_days, cities=args.cities)
except Exception as exc:
logger.exception("training settlement cycle failed: {}", exc)
elapsed = time.time() - started
wait_for = max(5.0, interval_sec - elapsed)
if _STOP_EVENT.wait(wait_for):
break
if __name__ == "__main__":
main()