Add external monitoring stack and alerting docs

scripts/alertmanager_telegram_relay.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import json
+import os
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Dict, List
+
+import requests
+
+
+def _chat_ids() -> List[str]:
+    raw = (
+        os.getenv("POLYWEATHER_MONITORING_ALERT_CHAT_IDS")
+        or os.getenv("TELEGRAM_CHAT_IDS")
+        or os.getenv("TELEGRAM_CHAT_ID")
+        or ""
+    )
+    return [item.strip() for item in raw.split(",") if item.strip()]
+
+
+def _format_alerts(payload: Dict[str, Any]) -> str:
+    alerts = payload.get("alerts") or []
+    if not isinstance(alerts, list) or not alerts:
+        return "PolyWeather monitoring received an empty alert payload."
+    lines = ["PolyWeather monitoring alert"]
+    for alert in alerts[:10]:
+        if not isinstance(alert, dict):
+            continue
+        status = str(alert.get("status") or "unknown").upper()
+        labels = alert.get("labels") or {}
+        annotations = alert.get("annotations") or {}
+        alert_name = labels.get("alertname") or "unknown_alert"
+        severity = labels.get("severity") or "info"
+        summary = annotations.get("summary") or annotations.get("description") or ""
+        lines.append(f"- [{status}] {alert_name} ({severity})")
+        if summary:
+            lines.append(f"  {summary}")
+    return "\n".join(lines)
+
+
+def _send_telegram_message(text: str) -> None:
+    token = str(os.getenv("TELEGRAM_BOT_TOKEN") or "").strip()
+    chat_ids = _chat_ids()
+    if not token or not chat_ids:
+        return
+    url = f"https://api.telegram.org/bot{token}/sendMessage"
+    for chat_id in chat_ids:
+        try:
+            requests.post(
+                url,
+                json={
+                    "chat_id": chat_id,
+                    "text": text,
+                    "disable_web_page_preview": True,
+                },
+                timeout=10,
+            ).raise_for_status()
+        except Exception:
+            continue
+
+
+class _Handler(BaseHTTPRequestHandler):
+    def do_GET(self) -> None:  # noqa: N802
+        if self.path.rstrip("/") == "/healthz":
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(b'{"ok":true}')
+            return
+        self.send_error(404)
+
+    def do_POST(self) -> None:  # noqa: N802
+        if self.path.rstrip("/") != "/alerts":
+            self.send_error(404)
+            return
+        length = int(self.headers.get("Content-Length", "0") or "0")
+        raw = self.rfile.read(length) if length > 0 else b"{}"
+        try:
+            payload = json.loads(raw.decode("utf-8"))
+        except Exception:
+            self.send_error(400, "invalid json")
+            return
+        if not isinstance(payload, dict):
+            self.send_error(400, "invalid payload")
+            return
+        _send_telegram_message(_format_alerts(payload))
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json; charset=utf-8")
+        self.end_headers()
+        self.wfile.write(b'{"ok":true}')
+
+    def log_message(self, format: str, *args: object) -> None:  # noqa: A003
+        return
+
+
+def main() -> None:
+    port = 9099
+    server = HTTPServer(("0.0.0.0", port), _Handler)
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

scripts/check_ops_health.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Dict, Tuple
+
+import requests
+
+
+def _get_json(url: str, timeout: float) -> Tuple[int, Dict]:
+    response = requests.get(url, timeout=timeout)
+    response.raise_for_status()
+    return response.status_code, response.json()
+
+
+def _get_text(url: str, timeout: float) -> Tuple[int, str]:
+    response = requests.get(url, timeout=timeout)
+    response.raise_for_status()
+    return response.status_code, response.text
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Run basic PolyWeather ops checks.")
+    parser.add_argument("--base-url", default="http://127.0.0.1:8000")
+    parser.add_argument("--timeout", type=float, default=8.0)
+    args = parser.parse_args()
+
+    base = args.base_url.rstrip("/")
+    timeout = args.timeout
+    report = {"checks": []}
+    failed = False
+
+    try:
+        _, health = _get_json(f"{base}/healthz", timeout)
+        ok = str(health.get("status") or "").lower() == "ok"
+        report["checks"].append({"name": "healthz", "ok": ok, "detail": health})
+        failed = failed or not ok
+    except Exception as exc:
+        report["checks"].append({"name": "healthz", "ok": False, "detail": str(exc)})
+        failed = True
+
+    try:
+        _, status = _get_json(f"{base}/api/system/status", timeout)
+        features = status.get("features") or {}
+        ok = status.get("status") == "ok" and bool((status.get("db") or {}).get("ok"))
+        report["checks"].append({"name": "system_status", "ok": ok, "detail": {"features": features, "db": status.get("db")}})
+        failed = failed or not ok
+    except Exception as exc:
+        report["checks"].append({"name": "system_status", "ok": False, "detail": str(exc)})
+        failed = True
+
+    try:
+        _, metrics = _get_text(f"{base}/metrics", timeout)
+        ok = "polyweather_http_requests_total" in metrics or "polyweather_source_requests_total" in metrics
+        report["checks"].append({"name": "metrics", "ok": ok, "detail": "metrics exposed"})
+        failed = failed or not ok
+    except Exception as exc:
+        report["checks"].append({"name": "metrics", "ok": False, "detail": str(exc)})
+        failed = True
+
+    print(json.dumps(report, ensure_ascii=False, indent=2))
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())