From 8835290acd6be3f9c39e47b8176489ff201e1788 Mon Sep 17 00:00:00 2001
From: Xuhui Zhu <simon.zhu@canonical.com>
Date: Tue, 9 Jul 2024 15:49:13 -0400
Subject: [PATCH] Add Prometheus alerting rules for Cinder agents state

The new rules use the metric below to check if any cinder service is disabled or down:
- openstack_cinder_agent_state

If any metric contains a disabled adminState, the rules will return warning.
If any metric results in 0, meaning service is down, the rules will return critical.

Change-Id: I77b9dd6d5fe18622998063be36a197a0aff83a10
---
 charms/openstack-exporter-k8s/README.md       |  8 +++++++
 .../prometheus_alert_rules/cinder_rules.yaml  | 24 +++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 charms/openstack-exporter-k8s/src/prometheus_alert_rules/cinder_rules.yaml

diff --git a/charms/openstack-exporter-k8s/README.md b/charms/openstack-exporter-k8s/README.md
index 2ecfb6d5..28d094d8 100644
--- a/charms/openstack-exporter-k8s/README.md
+++ b/charms/openstack-exporter-k8s/README.md
@@ -47,6 +47,14 @@ This charm automatically adds Prometheus alert rules using the files at
 `src/prometheus_alert_rules` when related with `grafana-agent`.
 The following alerts are configured by default:
 
+- `CinderStateWarning`: This alert rule will trigger when a cinder service is disabled. The
+exporter generates metric openstack_cinder_agent_state which checks cinder service status.
+Alerts will appear if any Cinder service is found to be disabled.
+
+- `CinderStateCritical`: This alert rule will trigger when a cinder service is down. The exporter
+generates metric openstack_cinder_agent_state which checks cinder service status.
+Alerts will appear if any Cinder service is found to be down.
+
 - `NeutronStateCritical`: This alert rule triggers when a Neutron agent is enabled, but down.
 The exporter generates the metric openstack_neutron_agent_state, which checks the status
 of neutron agents. Alerts will appear if any neutron agent is found to be down.
diff --git a/charms/openstack-exporter-k8s/src/prometheus_alert_rules/cinder_rules.yaml b/charms/openstack-exporter-k8s/src/prometheus_alert_rules/cinder_rules.yaml
new file mode 100644
index 00000000..514616b3
--- /dev/null
+++ b/charms/openstack-exporter-k8s/src/prometheus_alert_rules/cinder_rules.yaml
@@ -0,0 +1,24 @@
+groups:
+- name: Cinder
+  rules:
+    - alert: CinderStateWarning
+      expr: openstack_cinder_agent_state{adminState="disabled"}
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: Cinder service disabled. (Instance {{ $labels.hostname }})
+        description: |
+          The Cinder service is currently disabled on host {{ $labels.hostname }}.
+            LABELS = {{ $labels }}
+
+    - alert: CinderStateCritical
+      expr: openstack_cinder_agent_state{adminState="enabled"} == 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: Cinder service down. (Instance {{ $labels.hostname }})
+        description: |
+          The Cinder service is currently down on host {{ $labels.hostname }}.
+            LABELS = {{ $labels }}