Prometheus. Ceph. Monitoring and Alerting
Ceph monitoring
Links
// https://docs.ceph.com/en/latest/mgr/prometheus/
// https://grafana.com/grafana/dashboards/2842-ceph-cluster/
Ceph web
https://192.168.17.34:8443
Dashboard - Cluster - Services
Enable Prometheus at all nodes
Prometheus cli
nano /etc/prometheus/prometheus.yml
... # CEPH - job_name: 'ceph_node' file_sd_configs: - files: - ceph_node_targets.yml - job_name: 'ceph_ceph' honor_labels: true file_sd_configs: - files: - ceph_ceph_targets.yml
systemctl restart prometheus
Grafana web
Add new Dashboard
Import
-OR-
Import by ID 2842
CEPH ALERTING
----------
// https://github.com/jsuchenia/ceph-prometheus-rules/tree/master
PROMETHEUS CLI
---
nano /etc/prometheus/ceph_alert_rules.yml
---
groups:
- name: ceph.rules
rules:
- alert: CephTargetDown
expr: up{job="ceph"} == 0
for: 10m
labels:
severity: critical
annotations:
description: CEPH target down for more than 2m, please check - it could be a either exporter crash or a whole cluster crash
summary: CEPH exporter down
- alert: CephErrorState
expr: ceph_health_status > 1
for: 5m
labels:
severity: critical
annotations:
description: Ceph is in Error state longer than 5m, please check status of pools and OSDs
summary: CEPH in ERROR
- alert: CephWarnState
expr: ceph_health_status == 1
for: 30m
labels:
severity: warning
annotations:
description: Ceph is in Warn state longer than 30m, please check status of pools and OSDs
summary: CEPH in WARN
- alert: OsdDown
expr: ceph_osd_up == 0
for: 30m
labels:
severity: warning
annotations:
description: OSD is down longer than 30 min, please check whats the status
summary: OSD down
- alert: OsdApplyLatencyTooHigh
expr: ceph_osd_perf_apply_latency_seconds > 10
for: 90s
labels:
severity: warning
annotations:
description: OSD latency for {{ $labels.osd }} is too high. Please check if it doesnt stuck in weird state
summary: OSD latency too high {{ $labels.osd }}
- alert: MonitorClockSkewTooHigh
expr: abs(ceph_monitor_clock_skew_seconds) > 0.1
for: 60s
labels:
severity: warning
annotations:
description: Monitor clock skew detected on {{ $labels.monitor }} - please check ntp and harware clock settins
summary: Clock skew detected on {{ $labels.monitor }}
- alert: MonitorAvailableStorage
expr: ceph_monitor_avail_percent < 30
for: 60s
labels:
severity: warning
annotations:
description: Monitor storage for {{ $labels.monitor }} less than 30% - please check why its too high
summary: Nonitor storage for {{ $labels.monitor }} less than 30%
- alert: MonitorAvailableStorage
expr: ceph_monitor_avail_percent < 15
for: 60s
labels:
severity: critical
annotations:
description: Monitor storage for {{ $labels.monitor }} less than 15% - please check why its too high
summary: Nonitor storage for {{ $labels.monitor }} less than 15%
- alert: CephOSDUtilizatoin
expr: ceph_osd_utilization > 90
for: 60s
labels:
severity: critical
annotations:
description: Osd free space for {{ $labels.osd }} is higher tan 90%. Please validate why its so big, reweight or add storage
summary: OSD {{ $labels.osd }} is going out of space
- alert: CephPgDown
expr: ceph_pg_down > 0
for: 3m
labels:
severity: critical
annotations:
description: Some groups are down (unavailable) for too long on {{ $labels.cluster }}. Please ensure that all the data are available
summary: PG DOWN [{{ $value }}] on {{ $labels.cluster }}
- alert: CephPgIncomplete
expr: ceph_pg_incomplete > 0
for: 2m
labels:
severity: critical
annotations:
description: Some groups are incomplete (unavailable) for too long on {{ $labels.cluster }}. Please ensure that all the data are available
summary: PG INCOMPLETE [{{ $value }}] on {{ $labels.cluster }}
- alert: CephPgInconsistent
expr: ceph_pg_inconsistent > 0
for: 1m
labels:
severity: warning
annotations:
description: Some groups are inconsistent for too long on {{ $labels.cluster }}. Data is available but inconsistent across nodes
summary: PG INCONSISTENT [{{ $value }}] on {{ $labels.cluster }}
- alert: CephPgActivating
expr: ceph_pg_activating > 0
for: 5m
labels:
severity: critical
annotations:
description: Some groups are activating for too long on {{ $labels.cluster }}. Those PGs are unavailable for too long!
summary: PG ACTIVATING [{{ $value }}] on {{ $labels.cluster }}
- alert: CephPgBackfillTooFull
expr: ceph_pg_backfill_toofull > 0
for: 5m
labels:
severity: warning
annotations:
description: Some groups are located on full OSD on cluster {{ $labels.cluster }}. Those PGs can be unavailable shortly. Please check OSDs, change weight or reconfigure CRUSH rules.
summary: PG TOO FULL [{{ $value }}] on {{ $labels.cluster }}
- alert: CephPgUnavailable
expr: ceph_pg_total - ceph_pg_active > 0
for: 5m
labels:
severity: critical
annotations:
description: Some groups are unavailable on {{ $labels.cluster }}. Please check their detailed status and current configuration.
summary: PG UNAVAILABLE [{{ $value }}] on {{ $labels.cluster }}
- alert: CephOsdReweighted
expr: ceph_osd_weight < 1
for: 1h
labels:
severity: warning
annotations:
description: OSD {{ $labels.ceph_daemon}} on cluster {{ $labels.cluster}} was reweighted for too long. Please either create silent or fix that issue
summary: OSD {{ $labels.ceph_daemon }} on {{ $labels.cluster }} reweighted - {{ $value }}
---
nano /etc/prometheus/prometheus.yml
...
rule_files:
...
- "ceph_alert_rules.yml"
...
systemctl restart prometheus
PROMETHEUS WEB
---
http://<PROMETHEUS_IP>:9090/alerts?search=