From e8fb81d2c90bfac089ca61a98030f0f33846cbd0 Mon Sep 17 00:00:00 2001 From: Linda Guo Date: Thu, 8 Jul 2021 14:19:42 +1000 Subject: [PATCH] Rabbitmq metrics and splitbrain detection Enabled rabbitmq_prometheus plugin for prometheus to scrape the metrics of rabbitmq and alert if rabbitmq splitbrain is detected. Integrated rabbitmq dashboards in grafana via dashboards relations Added new unit test cases Closes-Bug: 1899183 Change-Id: I88942dd0b246c498d0ab40b00d586d4349b0f100 (cherry picked from commit 0653c186cecf720c522353da6169a2ecf05d3284) --- README.md | 18 + files/grafana-dashboard.json | 5841 +++++++++++++++++++++++ files/prom_rule_rmq_splitbrain.yaml | 10 + hooks/dashboards-relation-joined | 1 + hooks/prometheus-rules-relation-created | 1 + hooks/prometheus-rules-relation-joined | 1 + hooks/rabbit_utils.py | 9 +- hooks/rabbitmq_server_relations.py | 55 + hooks/scrape-relation-broken | 1 + hooks/scrape-relation-created | 1 + hooks/scrape-relation-joined | 1 + metadata.yaml | 6 + unit_tests/test_rabbit_utils.py | 19 + 13 files changed, 5961 insertions(+), 3 deletions(-) create mode 100644 files/grafana-dashboard.json create mode 100644 files/prom_rule_rmq_splitbrain.yaml create mode 120000 hooks/dashboards-relation-joined create mode 120000 hooks/prometheus-rules-relation-created create mode 120000 hooks/prometheus-rules-relation-joined create mode 120000 hooks/scrape-relation-broken create mode 120000 hooks/scrape-relation-created create mode 120000 hooks/scrape-relation-joined diff --git a/README.md b/README.md index bdd19d20..401614dd 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,24 @@ an application that supports the rabbitmq interface. For instance: juju add-relation rabbitmq-server:amqp nova-cloud-controller:amqp +## Monitoring + +To collect RabbitMQ metrics, add a relation between rabbitmq-server and +an application that supports the `scrape` interface. For instance: + + juju add-relation rabbitmq-server:scrape prometheus:scrape + +> **Note:** The scrape relation is only supported when the RabbitMQ version is >= 3.8. + +The charm can be related to a dashboard charm like grafana to view visualization metrics: + + juju add-relation rabbitmq-server:dashboards grafana:dashboards + +To get alerts of RabbitMQ split-brain events, add a relation between rabbitmq-server and +an application that supports the `prometheus-rules` interface. For instance: + + juju add-relation rabbitmq-server:prometheus-rules prometheus:prometheus-rules + ## High availability When more than one unit is deployed the charm will bring up a native RabbitMQ diff --git a/files/grafana-dashboard.json b/files/grafana-dashboard.json new file mode 100644 index 00000000..7474f4d7 --- /dev/null +++ b/files/grafana-dashboard.json @@ -0,0 +1,5841 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "prometheus", + "version": "2.0.0" + }, + { + "type": "table", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A new RabbitMQ Management Overview", + "editable": true, + "gnetId": 10991, + "graphTooltip": 1, + "iteration": 1610979946179, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Monitoring with Prometheus & Grafana", + "tooltip": "", + "type": "link", + "url": "https://www.rabbitmq.com/prometheus.html" + } + ], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": true, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#37872D", + "#1F60C4", + "#C4162A" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 64, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "10000,100000", + "timeFrom": null, + "timeShift": null, + "title": "Ready messages", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 62, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "-1,50", + "timeFrom": null, + "timeShift": null, + "title": "Incoming messages / s", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 66, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) - sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Publishers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 37, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_connections * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_queues * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Queues", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#37872D", + "#1F60C4", + "#C4162A" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 65, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "100,500", + "timeFrom": null, + "timeShift": null, + "title": "Unacknowledged messages", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 63, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "-1,50", + "timeFrom": null, + "timeShift": null, + "title": "Outgoing messages / s", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 41, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Consumers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 38, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Channels", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#1F60C4", + "#37872D", + "#C4162A" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 67, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "3,8", + "timeFrom": null, + "timeShift": null, + "title": "Nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 4, + "panels": [], + "title": "NODES", + "type": "row" + }, + { + "columns": [], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 69, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 8, + "desc": false + }, + "styles": [ + { + "alias": "Erlang/OTP", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "link": false, + "mappingType": 1, + "pattern": "erlang_version", + "thresholds": [ + "" + ], + "type": "string", + "unit": "none" + }, + { + "alias": "RabbitMQ", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "rabbitmq_version", + "thresholds": [ + "" + ], + "type": "string", + "unit": "none" + }, + { + "alias": "Host", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "instance", + "preserveFormat": false, + "thresholds": [], + "type": "string", + "unit": "short", + "valueMaps": [] + }, + { + "alias": "Node name", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "rabbitmq_node", + "thresholds": [ + "" + ], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Cluster", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "rabbitmq_cluster", + "thresholds": [], + "type": "hidden", + "unit": "short", + "valueMaps": [] + }, + { + "alias": "prometheus.erl", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "prometheus_client_version", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "rabbitmq_prometheus", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "prometheus_plugin_version", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transform": "table", + "type": "table-old" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_resident_memory_limit_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_resident_memory_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 536870912, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory available before publishers blocked", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_disk_space_available_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1073741824, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 5368709120, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk space available before publishers blocked", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 11 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_process_max_fds * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_fds * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 500, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 1000, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "File descriptors available", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 15 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_process_max_tcp_sockets * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_tcp_sockets * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 500, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 1000, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TCP sockets available", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 27, + "panels": [], + "title": "QUEUED MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages ready to be delivered to consumers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages pending consumer acknowledgement", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 11, + "panels": [], + "title": "INCOMING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages published / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_confirmed_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages confirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queue_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages routed to queues / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unconfirmed[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages unconfirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_dropped_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Unroutable messages dropped / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 36 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_returned_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Unroutable messages returned to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 29, + "panels": [], + "title": "OUTGOING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n (rate(rabbitmq_channel_messages_delivered_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\n (rate(rabbitmq_channel_messages_delivered_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages delivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 20, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 100, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages redelivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages delivered with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages delivered auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_acked_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages acknowledged / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 52 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Polling operations with auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_empty_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Polling operations that yield no result / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Polling operations with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 53, + "panels": [], + "title": "QUEUES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_queues * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total queues", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 63 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_declared_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queues declared / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 63 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_created_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queues created / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 63 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_deleted_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queues deleted / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 51, + "panels": [], + "title": "CHANNELS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_channels * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total channels", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 69 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_opened_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Channels opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 69 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_closed_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Channels closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 46, + "panels": [], + "title": "CONNECTIONS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_connections * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total connections", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 75 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_opened_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connections opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 75 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_closed_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connections closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "15s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "rabbitmq-prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 2, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource", + "datasource": "${DS_PROMETHEUS}" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(rabbitmq_identity_info, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(rabbitmq_identity_info, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "error": null, + "hide": 0, + "includeAll": false, + "label": "RabbitMQ Cluster", + "multi": false, + "name": "rabbitmq_cluster", + "options": [], + "query": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "15s", + "30s", + "1m", + "5m", + "10m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RabbitMQ-Overview", + "uid": "Kn5xm-gZk", + "version": 20210308, + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ] +} \ No newline at end of file diff --git a/files/prom_rule_rmq_splitbrain.yaml b/files/prom_rule_rmq_splitbrain.yaml new file mode 100644 index 00000000..4a58740a --- /dev/null +++ b/files/prom_rule_rmq_splitbrain.yaml @@ -0,0 +1,10 @@ + - alert: RabbitMQ_split_brain + # detect if rabbitmq_queues is different between rabbitmq nodes + expr: count(count(rabbitmq_queues) by (job)) > 1 + for: 5m + labels: + severity: page + application: rabbitmq-server + annotations: + description: RabbitMQ split brain detected + summary: RabbitMQ split brain detected diff --git a/hooks/dashboards-relation-joined b/hooks/dashboards-relation-joined new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/dashboards-relation-joined @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/prometheus-rules-relation-created b/hooks/prometheus-rules-relation-created new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/prometheus-rules-relation-created @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/prometheus-rules-relation-joined b/hooks/prometheus-rules-relation-joined new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/prometheus-rules-relation-joined @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/rabbit_utils.py b/hooks/rabbit_utils.py index 68d53ef7..5061679c 100644 --- a/hooks/rabbit_utils.py +++ b/hooks/rabbit_utils.py @@ -739,9 +739,12 @@ def get_plugin_manager(): :returns: Path to rabbitmq-plugins executable :rtype: str """ - manager = glob.glob( - '/usr/lib/rabbitmq/lib/rabbitmq_server-*/sbin/rabbitmq-plugins')[0] - return manager + # At version 3.8.2, only /sbin/rabbitmq-plugins can enable plugin correctly + if os.path.exists("/sbin/rabbitmq-plugins"): + return '/sbin/rabbitmq-plugins' + else: + return glob.glob( + '/usr/lib/rabbitmq/lib/rabbitmq_server-*/sbin/rabbitmq-plugins')[0] def _manage_plugin(plugin, action): diff --git a/hooks/rabbitmq_server_relations.py b/hooks/rabbitmq_server_relations.py index 53095999..252944f4 100755 --- a/hooks/rabbitmq_server_relations.py +++ b/hooks/rabbitmq_server_relations.py @@ -79,6 +79,7 @@ from charmhelpers.core.hookenv import ( DEBUG, ERROR, INFO, + WARNING, leader_set, leader_get, relation_get, @@ -254,6 +255,58 @@ def update_clients(check_deferred_restarts=True): check_deferred_restarts=check_deferred_restarts) +@hooks.hook('dashboards-relation-joined') +def dashboards_relation_joined(relation_id=None, remote_unit=None): + """ + dashboards relation joined + send the dashboard json data via relation + """ + with open(os.path.join("files", "grafana-dashboard.json")) as f: + dashboard_str = f.read() + relation_set(relation_id, relation_settings={"dashboard": dashboard_str, + "name": "RabbitMQ-Overview"}) + + +@hooks.hook('prometheus-rules-relation-joined', + 'prometheus-rules-relation-created') +def prometheus_rules_joined(relation_id=None, remote_unit=None): + """ + prometheus rules relation joined + send the prometheus rules via relation + """ + with open(os.path.join("files", "prom_rule_rmq_splitbrain.yaml")) as f: + rule = f.read() + relation_set(relation_id, relation_settings={"groups": rule}) + + +@hooks.hook('scrape-relation-joined', 'scrape-relation-created') +def prometheus_scrape_joined(relation_id=None, remote_unit=None): + """ + scrape relation joined + enable prometheus plugin and open port + """ + err_msg = "rabbitmq-server needs to be >= 3.8 to support Prometheus plugin" + if cmp_pkgrevno('rabbitmq-server', '3.8.0') < 0: + log(err_msg, level=WARNING) + status_set("blocked", err_msg) + raise Exception(err_msg) + rabbit.enable_plugin(PROM_PLUGIN) + open_port(RMQ_MON_PORT) + relation_set(relation_id, relation_settings={"port": RMQ_MON_PORT}) + + +@hooks.hook('scrape-relation-broken') +def prometheus_scape_broken(): + """ + scrape relation broken + the relation has been completely removed + disable prometheus plugin and close port + """ + rabbit.disable_plugin(PROM_PLUGIN) + close_port(RMQ_MON_PORT) + log("scrape relation broken, disabled plugin and close port", level=INFO) + + @validate_amqp_config_tracker @hooks.hook('amqp-relation-changed') def amqp_changed(relation_id=None, remote_unit=None, @@ -664,6 +717,8 @@ def upgrade_charm(): MAN_PLUGIN = 'rabbitmq_management' +PROM_PLUGIN = 'rabbitmq_prometheus' +RMQ_MON_PORT = 15692 @hooks.hook('config-changed') diff --git a/hooks/scrape-relation-broken b/hooks/scrape-relation-broken new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/scrape-relation-broken @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/scrape-relation-created b/hooks/scrape-relation-created new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/scrape-relation-created @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/scrape-relation-joined b/hooks/scrape-relation-joined new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/scrape-relation-joined @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 4876e931..d6c4ac10 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -22,6 +22,12 @@ provides: nrpe-external-master: interface: nrpe-external-master scope: container + dashboards: + interface: grafana-dashboard + scrape: + interface: http + prometheus-rules: + interface: prometheus-rules requires: ha: interface: hacluster diff --git a/unit_tests/test_rabbit_utils.py b/unit_tests/test_rabbit_utils.py index 08fefd8b..c77f1e92 100644 --- a/unit_tests/test_rabbit_utils.py +++ b/unit_tests/test_rabbit_utils.py @@ -893,6 +893,25 @@ class UtilsTests(CharmTestCase): mock_new_rabbitmq.return_value = True self.assertEqual(rabbit_utils.get_managment_port(), 15672) + @mock.patch('glob.glob') + @mock.patch('rabbit_utils.subprocess.check_call') + @mock.patch('os.path.exists') + def test_enable_management_plugin(self, mock_os_path, + mock_subprocess, + mock_glob): + mock_os_path.return_value = True + rabbitmq_plugins = '/sbin/rabbitmq-plugins' + rabbit_utils._manage_plugin("rabbitmq_prometheus", "enable") + mock_subprocess.assert_called_with([rabbitmq_plugins, + "enable", "rabbitmq_prometheus"]) + mock_os_path.return_value = False + rabbitmq_plugins = '/usr/lib/rabbitmq/lib/'\ + 'rabbitmq_server-3.8.2/sbin/rabbitmq-plugins' + mock_glob.return_value = [rabbitmq_plugins] + rabbit_utils._manage_plugin("rabbitmq_prometheus", "enable") + mock_subprocess.assert_called_with([rabbitmq_plugins, + "enable", "rabbitmq_prometheus"]) + @mock.patch('rabbit_utils.caching_cmp_pkgrevno') @mock.patch('rabbit_utils.relations_for_id') @mock.patch('rabbit_utils.subprocess')