From a8462e188bf6663ef29dcfc347dc5bc84755dda1 Mon Sep 17 00:00:00 2001 From: "2569718930@qq.com" <2569718930@qq.com> Date: Wed, 1 Apr 2026 01:49:58 +0800 Subject: [PATCH] Add external monitoring stack and alerting docs --- .env.example | 7 + README_ZH.md | 18 ++ docker-compose.yml | 66 +++++ docs/MONITORING_ZH.md | 123 ++++++++++ docs/OPS_ADMIN_ZH.md | 7 +- monitoring/alertmanager/alertmanager.yml | 15 ++ .../dashboards/polyweather-overview.json | 227 ++++++++++++++++++ .../provisioning/dashboards/dashboard.yml | 11 + .../provisioning/datasources/prometheus.yml | 9 + monitoring/prometheus/alerts.yml | 58 +++++ monitoring/prometheus/prometheus.yml | 19 ++ scripts/alertmanager_telegram_relay.py | 103 ++++++++ scripts/check_ops_health.py | 67 ++++++ 13 files changed, 729 insertions(+), 1 deletion(-) create mode 100644 docs/MONITORING_ZH.md create mode 100644 monitoring/alertmanager/alertmanager.yml create mode 100644 monitoring/grafana/dashboards/polyweather-overview.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboard.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus/alerts.yml create mode 100644 monitoring/prometheus/prometheus.yml create mode 100644 scripts/alertmanager_telegram_relay.py create mode 100644 scripts/check_ops_health.py diff --git a/.env.example b/.env.example index a77cdccc..87391f78 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,12 @@ OPEN_METEO_DISK_CACHE_PATH=/var/lib/polyweather/open_meteo_cache.json UID=1000 GID=1000 POLYWEATHER_STATE_STORAGE_MODE=dual +POLYWEATHER_PROMETHEUS_PORT=9090 +POLYWEATHER_ALERTMANAGER_PORT=9093 +POLYWEATHER_ALERT_RELAY_PORT=9099 +POLYWEATHER_GRAFANA_PORT=3001 +POLYWEATHER_GRAFANA_ADMIN_USER=admin +POLYWEATHER_GRAFANA_ADMIN_PASSWORD=polyweather ######################################## # 2) Telegram bot minimal @@ -72,6 +78,7 @@ TELEGRAM_ALERT_MIN_TRIGGER_COUNT=2 TELEGRAM_ALERT_MIN_SEVERITY=medium TELEGRAM_ALERT_MISPRICING_MAX_YES_BUY=0.10 TELEGRAM_ALERT_CITIES=ankara,london,paris,seoul,hong kong,shanghai,singapore,tokyo,tel aviv,toronto,buenos aires,wellington,new york,chicago,dallas,miami,atlanta,seattle,lucknow,sao paulo,munich +POLYWEATHER_MONITORING_ALERT_CHAT_IDS= ######################################## # 6) Frontend-facing shared values diff --git a/README_ZH.md b/README_ZH.md index 2edc54fc..ebf142ba 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -23,6 +23,7 @@ - 已上线支付运行态与审计接口：`/api/payments/runtime`。 - 已上线轻量运营后台：`/ops`（会员、周榜、补分、支付异常单）。 - 已上线轻量可观测性：`/healthz`、`/api/system/status`、`/metrics`。 +- 已补最小外部监控栈：Prometheus + Alertmanager + Grafana + Telegram 告警 relay。 - 运行态状态与缓存已支持 SQLite 渐进迁移：`file / dual / sqlite`。 - 已接入 EMOS/CRPS 校准链路，但当前仍保持 `emos_shadow`。 @@ -122,6 +123,22 @@ curl http://127.0.0.1:8000/metrics docker compose logs -f polyweather | egrep "payment event loop started|payment confirm loop started|payment auto-confirmed" ``` +### 外部监控栈 + +```bash +docker compose --profile monitoring up -d polyweather_prometheus polyweather_alertmanager polyweather_alert_relay polyweather_grafana +``` + +- Prometheus：`http://127.0.0.1:${POLYWEATHER_PROMETHEUS_PORT:-9090}` +- Alertmanager：`http://127.0.0.1:${POLYWEATHER_ALERTMANAGER_PORT:-9093}` +- Grafana：`http://127.0.0.1:${POLYWEATHER_GRAFANA_PORT:-3001}` + +手动巡检： + +```bash +python scripts/check_ops_health.py --base-url http://127.0.0.1:8000 +``` + ### 支付运行态 ```bash @@ -171,6 +188,7 @@ docker compose logs -f polyweather | egrep "polymarket wallet activity watcher s - 支付审计说明：[docs/payments/PAYMENT_AUDIT_ZH.md](docs/payments/PAYMENT_AUDIT_ZH.md) - 支付 V2 升级方案：[docs/payments/PAYMENT_UPGRADE_V2_ZH.md](docs/payments/PAYMENT_UPGRADE_V2_ZH.md) - 运营后台说明：[docs/OPS_ADMIN_ZH.md](docs/OPS_ADMIN_ZH.md) +- 外部监控说明：[docs/MONITORING_ZH.md](docs/MONITORING_ZH.md) - 深度评估报告：[docs/deep-research-report.md](docs/deep-research-report.md) - 前端报告：[FRONTEND_REDESIGN_REPORT.md](FRONTEND_REDESIGN_REPORT.md) - 发布流程：[RELEASE.md](RELEASE.md) diff --git a/docker-compose.yml b/docker-compose.yml index f1a94aca..8a5b7741 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,3 +33,69 @@ services: - "8000:8000" # UID/GID are mainly useful on Linux hosts to avoid root-owned output files. user: "${UID:-1000}:${GID:-1000}" + + polyweather_prometheus: + image: prom/prometheus:v3.4.1 + container_name: polyweather_prometheus + restart: unless-stopped + profiles: ["monitoring"] + depends_on: + - polyweather_web + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=15d" + - "--web.enable-lifecycle" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./monitoring/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}/monitoring/prometheus:/prometheus + ports: + - "${POLYWEATHER_PROMETHEUS_PORT:-9090}:9090" + + polyweather_alertmanager: + image: prom/alertmanager:v0.28.1 + container_name: polyweather_alertmanager + restart: unless-stopped + profiles: ["monitoring"] + depends_on: + - polyweather_alert_relay + command: + - "--config.file=/etc/alertmanager/alertmanager.yml" + - "--storage.path=/alertmanager" + volumes: + - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}/monitoring/alertmanager:/alertmanager + ports: + - "${POLYWEATHER_ALERTMANAGER_PORT:-9093}:9093" + + polyweather_alert_relay: + <<: *polyweather-base + container_name: polyweather_alert_relay + restart: unless-stopped + profiles: ["monitoring"] + command: python scripts/alertmanager_telegram_relay.py + volumes: + - ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}:/var/lib/polyweather + - ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}:/app/data + ports: + - "${POLYWEATHER_ALERT_RELAY_PORT:-9099}:9099" + user: "${UID:-1000}:${GID:-1000}" + + polyweather_grafana: + image: grafana/grafana-oss:12.0.2 + container_name: polyweather_grafana + restart: unless-stopped + profiles: ["monitoring"] + depends_on: + - polyweather_prometheus + environment: + GF_SECURITY_ADMIN_USER: ${POLYWEATHER_GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${POLYWEATHER_GRAFANA_ADMIN_PASSWORD:-polyweather} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ${POLYWEATHER_RUNTIME_DATA_DIR:-/var/lib/polyweather}/monitoring/grafana:/var/lib/grafana + ports: + - "${POLYWEATHER_GRAFANA_PORT:-3001}:3000" diff --git a/docs/MONITORING_ZH.md b/docs/MONITORING_ZH.md new file mode 100644 index 00000000..f781fc08 --- /dev/null +++ b/docs/MONITORING_ZH.md @@ -0,0 +1,123 @@ +# 外部监控与告警说明 + +最后更新：`2026-04-01` + +## 1. 目标 + +在现有轻量可观测性基础上，把 PolyWeather 补成最小可用的外部监控链路： + +- Prometheus 抓取 `/metrics` +- Alertmanager 根据规则聚合告警 +- Telegram relay 把告警推到运营频道 +- Grafana 展示趋势面板 +- 巡检脚本补健康检查 + +## 2. 组件 + +本仓库现在内置 4 个监控组件： + +- `polyweather_prometheus` +- `polyweather_alertmanager` +- `polyweather_alert_relay` +- `polyweather_grafana` + +对应配置目录： + +- [monitoring/prometheus/prometheus.yml](../monitoring/prometheus/prometheus.yml) +- [monitoring/prometheus/alerts.yml](../monitoring/prometheus/alerts.yml) +- [monitoring/alertmanager/alertmanager.yml](../monitoring/alertmanager/alertmanager.yml) +- [monitoring/grafana/dashboards/polyweather-overview.json](../monitoring/grafana/dashboards/polyweather-overview.json) + +## 3. 启动 + +```bash +docker compose --profile monitoring up -d polyweather_prometheus polyweather_alertmanager polyweather_alert_relay polyweather_grafana +``` + +默认端口： + +- Prometheus: `9090` +- Alertmanager: `9093` +- Grafana: `3001` +- Alert relay: `9099` + +## 4. 环境变量 + +在 [.env.example](../.env.example) 里新增了这些配置： + +```env +POLYWEATHER_PROMETHEUS_PORT=9090 +POLYWEATHER_ALERTMANAGER_PORT=9093 +POLYWEATHER_ALERT_RELAY_PORT=9099 +POLYWEATHER_GRAFANA_PORT=3001 +POLYWEATHER_GRAFANA_ADMIN_USER=admin +POLYWEATHER_GRAFANA_ADMIN_PASSWORD=polyweather +POLYWEATHER_MONITORING_ALERT_CHAT_IDS= +``` + +说明： + +- `POLYWEATHER_MONITORING_ALERT_CHAT_IDS` 为空时，relay 会自动回退到： + - `TELEGRAM_CHAT_IDS` + - `TELEGRAM_CHAT_ID` +- 告警发送仍复用现有 `TELEGRAM_BOT_TOKEN` + +## 5. 当前告警规则 + +当前默认规则： + +- `PolyWeatherWebDown` +- `PolyWeatherHttp5xxBurst` +- `PolyWeatherHighSourceErrorRate` +- `PolyWeatherOpenMeteoCooldownLoop` +- `PolyWeatherSlowHttpAverage` + +规则文件： + +- [monitoring/prometheus/alerts.yml](../monitoring/prometheus/alerts.yml) + +## 6. 当前 Grafana 面板 + +预置了一个最小仪表板： + +- `PolyWeather Overview` + +包含这些图： + +- HTTP Requests by Status +- HTTP Latency +- Source Requests by Outcome +- Source Error Rate (15m) + +## 7. 巡检脚本 + +手动巡检： + +```bash +python scripts/check_ops_health.py --base-url http://127.0.0.1:8000 +``` + +这个脚本会检查： + +- `/healthz` +- `/api/system/status` +- `/metrics` + +任何一项失败都会非零退出，适合挂到 crontab 或 systemd timer。 + +## 8. 备注 + +这套监控现在已经具备： + +- 外部抓取 +- 告警规则 +- Telegram 推送 +- 趋势面板 +- 巡检脚本 + +但它仍是“最小可用版”，还没有覆盖： + +- 节点级 CPU / 内存 / 磁盘 +- 数据库体积趋势 +- 更细粒度支付指标 +- 按城市/来源拆分的业务 SLA diff --git a/docs/OPS_ADMIN_ZH.md b/docs/OPS_ADMIN_ZH.md index 92986d35..727b955e 100644 --- a/docs/OPS_ADMIN_ZH.md +++ b/docs/OPS_ADMIN_ZH.md @@ -1,6 +1,6 @@ # Ops 运营后台说明 -最后更新：`2026-03-21` +最后更新：`2026-04-01` ## 1. 入口 @@ -28,6 +28,7 @@ POLYWEATHER_OPS_ADMIN_EMAILS=yhrsc30@gmail.com - 当前会员 - 周榜 - 支付异常单 +- 漏斗转化面板 ### 写能力 @@ -102,3 +103,7 @@ python scripts/reconcile_subscription_by_email.py --email - 让会员、积分、支付事故、系统状态可查 - 让常见人工操作不必再直接写 SQL + +外部监控与告警栈说明见： + +- [MONITORING_ZH.md](./MONITORING_ZH.md) diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 00000000..274b3e10 --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,15 @@ +global: + resolve_timeout: 5m + +route: + receiver: telegram-relay + group_by: ["alertname"] + group_wait: 30s + group_interval: 5m + repeat_interval: 3h + +receivers: + - name: telegram-relay + webhook_configs: + - url: http://polyweather_alert_relay:9099/alerts + send_resolved: true diff --git a/monitoring/grafana/dashboards/polyweather-overview.json b/monitoring/grafana/dashboards/polyweather-overview.json new file mode 100644 index 00000000..14bfe5b6 --- /dev/null +++ b/monitoring/grafana/dashboards/polyweather-overview.json @@ -0,0 +1,227 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "expr": "sum by (status) (rate(polyweather_http_requests_total[5m]))", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "HTTP Requests by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "expr": "sum(rate(polyweather_http_request_duration_ms_sum[5m])) / clamp_min(sum(rate(polyweather_http_request_duration_ms_count[5m])), 1)", + "legendFormat": "avg", + "refId": "A" + }, + { + "expr": "max(polyweather_http_request_duration_ms_max)", + "legendFormat": "max", + "refId": "B" + } + ], + "title": "HTTP Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "expr": "sum by (source, outcome) (rate(polyweather_source_requests_total[5m]))", + "legendFormat": "{{source}} / {{outcome}}", + "refId": "A" + } + ], + "title": "Source Requests by Outcome", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "targets": [ + { + "expr": "(sum(increase(polyweather_source_requests_total{outcome!~\"success|cache_hit\"}[15m])) / clamp_min(sum(increase(polyweather_source_requests_total[15m])), 1))", + "refId": "A" + } + ], + "title": "Source Error Rate (15m)", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 41, + "style": "dark", + "tags": [ + "polyweather", + "ops" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "PolyWeather Overview", + "uid": "polyweather-overview", + "version": 1 +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboard.yml b/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 00000000..a0db257f --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: PolyWeather Dashboards + orgId: 1 + folder: PolyWeather + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..f83c9fae --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: PolyWeather Prometheus + type: prometheus + access: proxy + url: http://polyweather_prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 00000000..e9775a8b --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,58 @@ +groups: + - name: polyweather-runtime + rules: + - alert: PolyWeatherWebDown + expr: up{job="polyweather-web"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "PolyWeather web metrics endpoint is down" + description: "/metrics on polyweather_web has been unreachable for more than 2 minutes." + + - alert: PolyWeatherHttp5xxBurst + expr: sum(increase(polyweather_http_requests_total{status=~"5.."}[10m])) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: "PolyWeather HTTP 5xx burst" + description: "More than 10 server errors were observed in the last 10 minutes." + + - alert: PolyWeatherHighSourceErrorRate + expr: | + ( + sum(increase(polyweather_source_requests_total{outcome!~"success|cache_hit"}[15m])) + / + clamp_min(sum(increase(polyweather_source_requests_total[15m])), 1) + ) > 0.25 + and sum(increase(polyweather_source_requests_total[15m])) > 20 + for: 10m + labels: + severity: warning + annotations: + summary: "PolyWeather source error rate is high" + description: "External weather source errors exceeded 25% over the last 15 minutes." + + - alert: PolyWeatherOpenMeteoCooldownLoop + expr: sum(increase(polyweather_source_requests_total{source="open_meteo",outcome=~"cooldown_skip|error"}[15m])) > 20 + for: 10m + labels: + severity: warning + annotations: + summary: "Open-Meteo is rate-limited or erroring" + description: "Open-Meteo has produced repeated cooldown skips or errors in the last 15 minutes." + + - alert: PolyWeatherSlowHttpAverage + expr: | + ( + sum(rate(polyweather_http_request_duration_ms_sum[10m])) + / + clamp_min(sum(rate(polyweather_http_request_duration_ms_count[10m])), 1) + ) > 2000 + for: 10m + labels: + severity: warning + annotations: + summary: "PolyWeather average HTTP latency is high" + description: "Average HTTP request duration exceeded 2s over the last 10 minutes." diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..b6563f44 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 30s + evaluation_interval: 30s + +rule_files: + - /etc/prometheus/alerts.yml + +alerting: + alertmanagers: + - static_configs: + - targets: + - polyweather_alertmanager:9093 + +scrape_configs: + - job_name: polyweather-web + metrics_path: /metrics + static_configs: + - targets: + - polyweather_web:8000 diff --git a/scripts/alertmanager_telegram_relay.py b/scripts/alertmanager_telegram_relay.py new file mode 100644 index 00000000..1788171f --- /dev/null +++ b/scripts/alertmanager_telegram_relay.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import json +import os +from http.server import BaseHTTPRequestHandler, HTTPServer +from typing import Any, Dict, List + +import requests + + +def _chat_ids() -> List[str]: + raw = ( + os.getenv("POLYWEATHER_MONITORING_ALERT_CHAT_IDS") + or os.getenv("TELEGRAM_CHAT_IDS") + or os.getenv("TELEGRAM_CHAT_ID") + or "" + ) + return [item.strip() for item in raw.split(",") if item.strip()] + + +def _format_alerts(payload: Dict[str, Any]) -> str: + alerts = payload.get("alerts") or [] + if not isinstance(alerts, list) or not alerts: + return "PolyWeather monitoring received an empty alert payload." + lines = ["PolyWeather monitoring alert"] + for alert in alerts[:10]: + if not isinstance(alert, dict): + continue + status = str(alert.get("status") or "unknown").upper() + labels = alert.get("labels") or {} + annotations = alert.get("annotations") or {} + alert_name = labels.get("alertname") or "unknown_alert" + severity = labels.get("severity") or "info" + summary = annotations.get("summary") or annotations.get("description") or "" + lines.append(f"- [{status}] {alert_name} ({severity})") + if summary: + lines.append(f" {summary}") + return "\n".join(lines) + + +def _send_telegram_message(text: str) -> None: + token = str(os.getenv("TELEGRAM_BOT_TOKEN") or "").strip() + chat_ids = _chat_ids() + if not token or not chat_ids: + return + url = f"https://api.telegram.org/bot{token}/sendMessage" + for chat_id in chat_ids: + try: + requests.post( + url, + json={ + "chat_id": chat_id, + "text": text, + "disable_web_page_preview": True, + }, + timeout=10, + ).raise_for_status() + except Exception: + continue + + +class _Handler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + if self.path.rstrip("/") == "/healthz": + self.send_response(200) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.end_headers() + self.wfile.write(b'{"ok":true}') + return + self.send_error(404) + + def do_POST(self) -> None: # noqa: N802 + if self.path.rstrip("/") != "/alerts": + self.send_error(404) + return + length = int(self.headers.get("Content-Length", "0") or "0") + raw = self.rfile.read(length) if length > 0 else b"{}" + try: + payload = json.loads(raw.decode("utf-8")) + except Exception: + self.send_error(400, "invalid json") + return + if not isinstance(payload, dict): + self.send_error(400, "invalid payload") + return + _send_telegram_message(_format_alerts(payload)) + self.send_response(200) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.end_headers() + self.wfile.write(b'{"ok":true}') + + def log_message(self, format: str, *args: object) -> None: # noqa: A003 + return + + +def main() -> None: + port = 9099 + server = HTTPServer(("0.0.0.0", port), _Handler) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/scripts/check_ops_health.py b/scripts/check_ops_health.py new file mode 100644 index 00000000..c752cccf --- /dev/null +++ b/scripts/check_ops_health.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import argparse +import json +import sys +from typing import Dict, Tuple + +import requests + + +def _get_json(url: str, timeout: float) -> Tuple[int, Dict]: + response = requests.get(url, timeout=timeout) + response.raise_for_status() + return response.status_code, response.json() + + +def _get_text(url: str, timeout: float) -> Tuple[int, str]: + response = requests.get(url, timeout=timeout) + response.raise_for_status() + return response.status_code, response.text + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run basic PolyWeather ops checks.") + parser.add_argument("--base-url", default="http://127.0.0.1:8000") + parser.add_argument("--timeout", type=float, default=8.0) + args = parser.parse_args() + + base = args.base_url.rstrip("/") + timeout = args.timeout + report = {"checks": []} + failed = False + + try: + _, health = _get_json(f"{base}/healthz", timeout) + ok = str(health.get("status") or "").lower() == "ok" + report["checks"].append({"name": "healthz", "ok": ok, "detail": health}) + failed = failed or not ok + except Exception as exc: + report["checks"].append({"name": "healthz", "ok": False, "detail": str(exc)}) + failed = True + + try: + _, status = _get_json(f"{base}/api/system/status", timeout) + features = status.get("features") or {} + ok = status.get("status") == "ok" and bool((status.get("db") or {}).get("ok")) + report["checks"].append({"name": "system_status", "ok": ok, "detail": {"features": features, "db": status.get("db")}}) + failed = failed or not ok + except Exception as exc: + report["checks"].append({"name": "system_status", "ok": False, "detail": str(exc)}) + failed = True + + try: + _, metrics = _get_text(f"{base}/metrics", timeout) + ok = "polyweather_http_requests_total" in metrics or "polyweather_source_requests_total" in metrics + report["checks"].append({"name": "metrics", "ok": ok, "detail": "metrics exposed"}) + failed = failed or not ok + except Exception as exc: + report["checks"].append({"name": "metrics", "ok": False, "detail": str(exc)}) + failed = True + + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())