Prometheus. Ceph. Monitoring and Alerting

Ceph monitoring

Links

// https://docs.ceph.com/en/latest/mgr/prometheus/

// https://grafana.com/grafana/dashboards/2842-ceph-cluster/

Ceph web

https://192.168.17.34:8443

Dashboard - Cluster - Services

         Enable Prometheus at all nodes

Prometheus cli

nano /etc/prometheus/prometheus.yml

...
  # CEPH
  - job_name: 'ceph_node'
    file_sd_configs:
      - files:
        - ceph_node_targets.yml
  - job_name: 'ceph_ceph'
    honor_labels: true
    file_sd_configs:
      - files:        - ceph_ceph_targets.yml

systemctl restart prometheus

Grafana web

Add new Dashboard

  Import

   -OR-

  Import by ID 2842

CEPH ALERTING

----------

// https://github.com/jsuchenia/ceph-prometheus-rules/tree/master

PROMETHEUS CLI

---

nano /etc/prometheus/ceph_alert_rules.yml

---

groups:

- name: ceph.rules

  rules:

  - alert: CephTargetDown

   expr: up{job="ceph"} == 0

   for: 10m

   labels:

     severity: critical

   annotations:

     description: CEPH target down for more than 2m, please check - it could be a either exporter crash or a whole cluster crash

     summary: CEPH exporter down

  - alert: CephErrorState

   expr: ceph_health_status > 1

   for: 5m

   labels:

     severity: critical

   annotations:

     description: Ceph is in Error state longer than 5m, please check status of pools and OSDs

     summary: CEPH in ERROR

  - alert: CephWarnState

   expr: ceph_health_status == 1

   for: 30m

   labels:

     severity: warning

   annotations:

     description: Ceph is in Warn state longer than 30m, please check status of pools and OSDs

     summary: CEPH in WARN

  - alert: OsdDown

   expr: ceph_osd_up == 0

   for: 30m

   labels:

     severity: warning

   annotations:

     description: OSD is down longer than 30 min, please check whats the status

     summary: OSD down

  - alert: OsdApplyLatencyTooHigh

   expr: ceph_osd_perf_apply_latency_seconds > 10

   for: 90s

   labels:

     severity: warning

   annotations:

     description: OSD latency for {{ $labels.osd }} is too high. Please check if it doesnt stuck in weird state

     summary: OSD latency too high {{ $labels.osd }}

  - alert: MonitorClockSkewTooHigh

   expr: abs(ceph_monitor_clock_skew_seconds) > 0.1

   for: 60s

   labels:

     severity: warning

   annotations:

     description: Monitor clock skew detected on  {{ $labels.monitor }} - please check ntp and harware clock settins

     summary: Clock skew detected on {{ $labels.monitor }}

  - alert: MonitorAvailableStorage

   expr: ceph_monitor_avail_percent < 30

   for: 60s

   labels:

     severity: warning

   annotations:

     description: Monitor storage for {{ $labels.monitor }} less than 30% - please check why its too high

     summary: Nonitor storage for  {{ $labels.monitor }} less than 30%

  - alert: MonitorAvailableStorage

   expr: ceph_monitor_avail_percent < 15

   for: 60s

   labels:

     severity: critical

   annotations:

     description: Monitor storage for {{ $labels.monitor }} less than 15% - please check why its too high

     summary: Nonitor storage for  {{ $labels.monitor }} less than 15%

  - alert: CephOSDUtilizatoin

   expr: ceph_osd_utilization > 90

   for: 60s

   labels:

     severity: critical

   annotations:

     description: Osd free space for  {{ $labels.osd }} is higher tan 90%. Please validate why its so big, reweight or add storage

     summary: OSD {{ $labels.osd }} is going out of space

  - alert: CephPgDown

   expr: ceph_pg_down > 0

   for: 3m

   labels:

     severity: critical

   annotations:

     description: Some groups are down (unavailable) for too long on {{ $labels.cluster }}. Please ensure that all the data are available

     summary: PG DOWN [{{ $value }}] on {{ $labels.cluster }}

  - alert: CephPgIncomplete

   expr: ceph_pg_incomplete > 0

   for: 2m

   labels:

     severity: critical

   annotations:

     description: Some groups are incomplete (unavailable) for too long on {{ $labels.cluster }}. Please ensure that all the data are available

     summary: PG INCOMPLETE [{{ $value }}] on {{ $labels.cluster }}

  - alert: CephPgInconsistent

   expr: ceph_pg_inconsistent > 0

   for: 1m

   labels:

     severity: warning

   annotations:

     description: Some groups are inconsistent for too long on {{ $labels.cluster }}. Data is available but inconsistent across nodes

     summary: PG INCONSISTENT [{{ $value }}] on {{ $labels.cluster }}

  - alert: CephPgActivating

   expr: ceph_pg_activating > 0

   for: 5m

   labels:

     severity: critical

   annotations:

     description: Some groups are activating for too long on {{ $labels.cluster }}. Those PGs are unavailable for too long!

     summary: PG ACTIVATING [{{ $value }}] on {{ $labels.cluster }}

  - alert: CephPgBackfillTooFull

   expr: ceph_pg_backfill_toofull > 0

   for: 5m

   labels:

     severity: warning

   annotations:

     description: Some groups are located on full OSD on cluster {{ $labels.cluster }}. Those PGs can be unavailable shortly. Please check OSDs, change weight or reconfigure CRUSH rules.

     summary: PG TOO FULL [{{ $value }}] on {{ $labels.cluster }}

  - alert: CephPgUnavailable

   expr: ceph_pg_total - ceph_pg_active > 0

   for: 5m

   labels:

     severity: critical

   annotations:

     description: Some groups are unavailable on {{ $labels.cluster }}. Please check their detailed status and current configuration.

     summary: PG UNAVAILABLE [{{ $value }}] on {{ $labels.cluster }}

  - alert: CephOsdReweighted

   expr: ceph_osd_weight < 1

   for: 1h

   labels:

     severity: warning

   annotations:

     description: OSD {{ $labels.ceph_daemon}} on cluster {{ $labels.cluster}} was reweighted for too long. Please either create silent or fix that issue

     summary: OSD {{ $labels.ceph_daemon }} on {{ $labels.cluster }} reweighted - {{ $value }}

---

nano /etc/prometheus/prometheus.yml

...

rule_files:

  ...

  - "ceph_alert_rules.yml"

...

systemctl restart prometheus

PROMETHEUS WEB

---

http://<PROMETHEUS_IP>:9090/alerts?search=