Add external monitoring stack and alerting docs
This commit is contained in:
2569718930@qq.com
@@ -0,0 +1,15 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
|
||||
route:
|
||||
receiver: telegram-relay
|
||||
group_by: ["alertname"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 3h
|
||||
|
||||
receivers:
|
||||
- name: telegram-relay
|
||||
webhook_configs:
|
||||
- url: http://polyweather_alert_relay:9099/alerts
|
||||
send_resolved: true
|
||||
@@ -0,0 +1,227 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (status) (rate(polyweather_http_requests_total[5m]))",
|
||||
"legendFormat": "{{status}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "HTTP Requests by Status",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"unit": "ms"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(polyweather_http_request_duration_ms_sum[5m])) / clamp_min(sum(rate(polyweather_http_request_duration_ms_count[5m])), 1)",
|
||||
"legendFormat": "avg",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "max(polyweather_http_request_duration_ms_max)",
|
||||
"legendFormat": "max",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "HTTP Latency",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (source, outcome) (rate(polyweather_source_requests_total[5m]))",
|
||||
"legendFormat": "{{source}} / {{outcome}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Source Requests by Outcome",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 10
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 25
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(sum(increase(polyweather_source_requests_total{outcome!~\"success|cache_hit\"}[15m])) / clamp_min(sum(increase(polyweather_source_requests_total[15m])), 1))",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Source Error Rate (15m)",
|
||||
"type": "stat"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 41,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"polyweather",
|
||||
"ops"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "PolyWeather Overview",
|
||||
"uid": "polyweather-overview",
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: PolyWeather Dashboards
|
||||
orgId: 1
|
||||
folder: PolyWeather
|
||||
type: file
|
||||
disableDeletion: false
|
||||
editable: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: PolyWeather Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://polyweather_prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1,58 @@
|
||||
groups:
|
||||
- name: polyweather-runtime
|
||||
rules:
|
||||
- alert: PolyWeatherWebDown
|
||||
expr: up{job="polyweather-web"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "PolyWeather web metrics endpoint is down"
|
||||
description: "/metrics on polyweather_web has been unreachable for more than 2 minutes."
|
||||
|
||||
- alert: PolyWeatherHttp5xxBurst
|
||||
expr: sum(increase(polyweather_http_requests_total{status=~"5.."}[10m])) > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PolyWeather HTTP 5xx burst"
|
||||
description: "More than 10 server errors were observed in the last 10 minutes."
|
||||
|
||||
- alert: PolyWeatherHighSourceErrorRate
|
||||
expr: |
|
||||
(
|
||||
sum(increase(polyweather_source_requests_total{outcome!~"success|cache_hit"}[15m]))
|
||||
/
|
||||
clamp_min(sum(increase(polyweather_source_requests_total[15m])), 1)
|
||||
) > 0.25
|
||||
and sum(increase(polyweather_source_requests_total[15m])) > 20
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PolyWeather source error rate is high"
|
||||
description: "External weather source errors exceeded 25% over the last 15 minutes."
|
||||
|
||||
- alert: PolyWeatherOpenMeteoCooldownLoop
|
||||
expr: sum(increase(polyweather_source_requests_total{source="open_meteo",outcome=~"cooldown_skip|error"}[15m])) > 20
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Open-Meteo is rate-limited or erroring"
|
||||
description: "Open-Meteo has produced repeated cooldown skips or errors in the last 15 minutes."
|
||||
|
||||
- alert: PolyWeatherSlowHttpAverage
|
||||
expr: |
|
||||
(
|
||||
sum(rate(polyweather_http_request_duration_ms_sum[10m]))
|
||||
/
|
||||
clamp_min(sum(rate(polyweather_http_request_duration_ms_count[10m])), 1)
|
||||
) > 2000
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PolyWeather average HTTP latency is high"
|
||||
description: "Average HTTP request duration exceeded 2s over the last 10 minutes."
|
||||
@@ -0,0 +1,19 @@
|
||||
global:
|
||||
scrape_interval: 30s
|
||||
evaluation_interval: 30s
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yml
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- polyweather_alertmanager:9093
|
||||
|
||||
scrape_configs:
|
||||
- job_name: polyweather-web
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets:
|
||||
- polyweather_web:8000
|
||||
Reference in New Issue
Block a user