diff --git a/README.md b/README.md index bdd19d20..401614dd 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,24 @@ an application that supports the rabbitmq interface. For instance: juju add-relation rabbitmq-server:amqp nova-cloud-controller:amqp +## Monitoring + +To collect RabbitMQ metrics, add a relation between rabbitmq-server and +an application that supports the `scrape` interface. For instance: + + juju add-relation rabbitmq-server:scrape prometheus:scrape + +> **Note:** The scrape relation is only supported when the RabbitMQ version is >= 3.8. + +The charm can be related to a dashboard charm like grafana to view visualization metrics: + + juju add-relation rabbitmq-server:dashboards grafana:dashboards + +To get alerts of RabbitMQ split-brain events, add a relation between rabbitmq-server and +an application that supports the `prometheus-rules` interface. For instance: + + juju add-relation rabbitmq-server:prometheus-rules prometheus:prometheus-rules + ## High availability When more than one unit is deployed the charm will bring up a native RabbitMQ diff --git a/files/grafana-dashboard.json b/files/grafana-dashboard.json new file mode 100644 index 00000000..7474f4d7 --- /dev/null +++ b/files/grafana-dashboard.json @@ -0,0 +1,5841 @@ +{ + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "7.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "prometheus", + "version": "2.0.0" + }, + { + "type": "table", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A new RabbitMQ Management Overview", + "editable": true, + "gnetId": 10991, + "graphTooltip": 1, + "iteration": 1610979946179, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Monitoring with Prometheus & Grafana", + "tooltip": "", + "type": "link", + "url": "https://www.rabbitmq.com/prometheus.html" + } + ], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": true, + "colorPrefix": false, + "colorValue": false, + "colors": [ + "#37872D", + "#1F60C4", + "#C4162A" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 64, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "10000,100000", + "timeFrom": null, + "timeShift": null, + "title": "Ready messages", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 62, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "-1,50", + "timeFrom": null, + "timeShift": null, + "title": "Incoming messages / s", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 66, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) - sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Publishers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 37, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_connections * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_queues * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Queues", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#37872D", + "#1F60C4", + "#C4162A" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 65, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "100,500", + "timeFrom": null, + "timeShift": null, + "title": "Unacknowledged messages", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 63, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_messages_delivered_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\nsum(rate(rabbitmq_channel_get_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "-1,50", + "timeFrom": null, + "timeShift": null, + "title": "Outgoing messages / s", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 3 + }, + "id": 41, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_channel_consumers * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Consumers", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#1F60C4", + "#37872D" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "id": 38, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_channels * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "0,10", + "timeFrom": null, + "timeShift": null, + "title": "Channels", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#1F60C4", + "#37872D", + "#C4162A" + ], + "datasource": "${DS_PROMETHEUS}", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "id": 67, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.1.3", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(255, 255, 255, 0)", + "full": false, + "lineColor": "rgb(255, 255, 255)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "3,8", + "timeFrom": null, + "timeShift": null, + "title": "Nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 4, + "panels": [], + "title": "NODES", + "type": "row" + }, + { + "columns": [], + "datasource": "${DS_PROMETHEUS}", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 69, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 8, + "desc": false + }, + "styles": [ + { + "alias": "Erlang/OTP", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "link": false, + "mappingType": 1, + "pattern": "erlang_version", + "thresholds": [ + "" + ], + "type": "string", + "unit": "none" + }, + { + "alias": "RabbitMQ", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "rabbitmq_version", + "thresholds": [ + "" + ], + "type": "string", + "unit": "none" + }, + { + "alias": "Host", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "instance", + "preserveFormat": false, + "thresholds": [], + "type": "string", + "unit": "short", + "valueMaps": [] + }, + { + "alias": "Node name", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "rabbitmq_node", + "thresholds": [ + "" + ], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Cluster", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "rabbitmq_cluster", + "thresholds": [], + "type": "hidden", + "unit": "short", + "valueMaps": [] + }, + { + "alias": "prometheus.erl", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "prometheus_client_version", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "rabbitmq_prometheus", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "prometheus_plugin_version", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "rabbitmq_build_info * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transform": "table", + "type": "table-old" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "If the value is zero or less, the memory alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the memory alarm is triggered with a slight delay.\n\nThe kernel's view of the amount of memory used by the node can differ from what the node itself can observe. This means that this value can be negative for a sustained period of time.\n\nBy default nodes use resident set size (RSS) to compute how much memory they use. This strategy can be changed (see the guides below).\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Memory Alarms](https://www.rabbitmq.com/memory.html)\n* [Reasoning About Memory Use](https://www.rabbitmq.com/memory-use.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_resident_memory_limit_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_resident_memory_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 536870912, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory available before publishers blocked", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "This metric is reported for the partition where the RabbitMQ data directory is stored.\n\nIf the value is zero or less, the disk alarm will be triggered and all publishing connections across all cluster nodes will be blocked.\n\nThis value can temporarily go negative because the free disk space alarm is triggered with a slight delay.\n\n* [Alarms](https://www.rabbitmq.com/alarms.html)\n* [Disk Space Alarms](https://www.rabbitmq.com/disk-alarms.html)\n* [Disk Space](https://www.rabbitmq.com/production-checklist.html#resource-limits-disk-space)\n* [Persistence Configuration](https://www.rabbitmq.com/persistence-conf.html)\n* [Blocked Connection Notifications](https://www.rabbitmq.com/connection-blocked.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_disk_space_available_bytes * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1073741824, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 5368709120, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk space available before publishers blocked", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "When this value reaches zero, new connections will not be accepted and disk write operations may fail.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Open File Handles Limit](https://www.rabbitmq.com/production-checklist.html#resource-limits-file-handle-limit)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 11 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_process_max_fds * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_fds * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 500, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 1000, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "File descriptors available", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "When this value reaches zero, new connections will not be accepted.\n\nClient libraries, peer nodes and CLI tools will not be able to connect when the node runs out of available file descriptors.\n\n* [Networking and RabbitMQ](https://www.rabbitmq.com/networking.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 15 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(rabbitmq_process_max_tcp_sockets * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) -\n(rabbitmq_process_open_tcp_sockets * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 500, + "yaxis": "left" + }, + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "lt", + "value": 1000, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TCP sockets available", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 27, + "panels": [], + "title": "QUEUED MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages ready to be delivered to consumers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The total number of messages that are either in-flight to consumers, currently being processed by consumers or simply waiting for the consumer acknowledgements to be processed by the queue. Until the queue processes the message acknowledgement, the message will remain unacknowledged.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_unacked * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages pending consumer acknowledgement", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 11, + "panels": [], + "title": "INCOMING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages published / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages confirmed by the broker to publishers. Publishers must opt-in to receive message confirmations.\n\nIf this metric is consistently at zero it may suggest that publisher confirms are not used by clients. The safety of published messages is likely to be at risk.\n\n* [Publisher Confirms](https://www.rabbitmq.com/confirms.html#publisher-confirms)\n* [Publisher Confirms and Data Safety](https://www.rabbitmq.com/publishers.html#data-safety)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_confirmed_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages confirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queue_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages routed to queues / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages received from publishers that have publisher confirms enabled and the broker has not confirmed yet.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)\n* [Confirms and Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 31 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unconfirmed[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages unconfirmed to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages that cannot be routed and are dropped. \n\nAny value above zero means message loss and likely suggests a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_dropped_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Unroutable messages dropped / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages that cannot be routed and are returned back to publishers.\n\nSustained values above zero may indicate a routing problem on the publisher end.\n\n* [Unroutable Message Handling](https://www.rabbitmq.com/publishers.html#unroutable)\n* [When Will Published Messages Be Confirmed by the Broker?](https://www.rabbitmq.com/confirms.html#when-publishes-are-confirmed)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 36 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_unroutable_returned_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Unroutable messages returned to publishers / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 29, + "panels": [], + "title": "OUTGOING MESSAGES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages delivered to consumers. It includes messages that have been redelivered.\n\nThis metric does not include messages that have been fetched by consumers using `basic.get` (consumed by polling).\n\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n (rate(rabbitmq_channel_messages_delivered_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) +\n (rate(rabbitmq_channel_messages_delivered_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"})\n) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages delivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages that have been redelivered to consumers. It includes messages that have been requeued automatically and redelivered due to channel exceptions or connection closures.\n\nHaving some redeliveries is expected, but if this metric is consistently non-zero, it is worth investigating why.\n\n* [Negative Acknowledgement and Requeuing of Deliveries](https://www.rabbitmq.com/confirms.html#consumer-nacks-requeue)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_redelivered_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 20, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 100, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages redelivered / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of message deliveries to consumers that use manual acknowledgement mode.\n\nWhen this mode is used, RabbitMQ waits for consumers to acknowledge messages before more messages can be delivered.\n\nThis is the safest way of consuming messages.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 47 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages delivered with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of message deliveries to consumers that use automatic acknowledgement mode.\n\nWhen this mode is used, RabbitMQ does not wait for consumers to acknowledge message deliveries.\n\nThis mode is fire-and-forget and does not offer any delivery safety guarantees. It tends to provide higher throughput and it may lead to consumer overload and higher consumer memory usage.\n\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 47 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_delivered_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages delivered auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of message acknowledgements coming from consumers that use manual acknowledgement mode.\n\n* [Consumer Acknowledgements](https://www.rabbitmq.com/confirms.html)\n* [Consumer Prefetch](https://www.rabbitmq.com/consumer-prefetch.html)\n* [Consumer Acknowledgement Modes, Prefetch and Throughput](https://www.rabbitmq.com/confirms.html#channel-qos-prefetch-throughput)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 52 + }, + "hiddenSeries": false, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_acked_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Messages acknowledged / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages delivered to polling consumers that use automatic acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 52 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Polling operations with auto ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of polling consumer operations that yield no result.\n\nAny value above zero means that RabbitMQ resources are wasted by polling consumers.\n\nCompare this metric to the other polling consumer metrics to see the inefficiency rate.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 57 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_empty_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Polling operations that yield no result / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of messages delivered to polling consumers that use manual acknowledgement mode.\n\nThe use of polling consumers is highly inefficient and therefore strongly discouraged.\n\n* [Fetching individual messages](https://www.rabbitmq.com/consumers.html#fetching)\n* [Consumers](https://www.rabbitmq.com/consumers.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 57 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/rabbit/", + "color": "#C4162A" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_get_ack_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 0, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Polling operations with manual ack / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 53, + "panels": [], + "title": "QUEUES", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of queue masters per node. \n\nThis metric makes it easy to see sub-optimal queue distribution in a cluster.\n\n* [Queue Masters, Data Locality](https://www.rabbitmq.com/ha.html#master-migration-data-locality)\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_queues * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total queues", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of queue declarations performed by clients.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 63 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_declared_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queues declared / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of new queues created (as opposed to redeclarations).\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 63 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_created_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queues created / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of queues deleted.\n\nLow sustained values above zero are to be expected. High rates may be indicative of queue churn or high rates of connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [Queues](https://www.rabbitmq.com/queues.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 63 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queues_deleted_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queues deleted / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 51, + "panels": [], + "title": "CHANNELS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of channels on all currently opened connections.\n\nIf this metric grows monotonically it is highly likely a channel leak in one of the applications. Confirm channel leaks by using the _Channels opened_ and _Channels closed_ metrics.\n\n* [Channel Leak](https://www.rabbitmq.com/channels.html#channel-leaks)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 69 + }, + "hiddenSeries": false, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_channels * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total channels", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of new channels opened by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 69 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_opened_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Channels opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of channels closed by applications across all connections. Channels are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of channel churn or mass connection recovery. Confirm connection recovery rates by using the _Connections opened_ metric.\n\n* [High Channel Churn](https://www.rabbitmq.com/channels.html#high-channel-churn)\n* [Channels](https://www.rabbitmq.com/channels.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 69 + }, + "hiddenSeries": false, + "id": 56, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channels_closed_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Channels closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 46, + "panels": [], + "title": "CONNECTIONS", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "Total number of client connections.\n\nIf this metric grows monotonically it is highly likely a connection leak in one of the applications. Confirm connection leaks by using the _Connections opened_ and _Connections closed_ metrics.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 75 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_connections * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total connections", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": -1, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of new connections opened by clients. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connection Leak](https://www.rabbitmq.com/connections.html#monitoring)\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 75 + }, + "hiddenSeries": false, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_opened_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connections opened / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "The rate of connections closed. Connections are expected to be long-lived.\n\nLow sustained values above zero are to be expected. High rates may be indicative of connection churn or mass connection recovery.\n\n* [Connections](https://www.rabbitmq.com/connections.html)", + "fieldConfig": { + "defaults": { + "custom": {}, + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 75 + }, + "hiddenSeries": false, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.2", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?0(\\b|\\.)/", + "color": "#56A64B" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?1(\\b|\\.)/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?2(\\b|\\.)/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?3(\\b|\\.)/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?4(\\b|\\.)/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?5(\\b|\\.)/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?6(\\b|\\.)/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?7(\\b|\\.)/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?8(\\b|\\.)/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[a-zA-Z\\.\\-]*?9(\\b|\\.)/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_connections_closed_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\", namespace=\"$namespace\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [ + { + "colorMode": "warning", + "fill": true, + "line": true, + "op": "gt", + "value": 2, + "yaxis": "left" + }, + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 10, + "yaxis": "left" + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connections closed / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "15s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "rabbitmq-prometheus" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 2, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource", + "datasource": "${DS_PROMETHEUS}" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(rabbitmq_identity_info, namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(rabbitmq_identity_info, namespace)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "error": null, + "hide": 0, + "includeAll": false, + "label": "RabbitMQ Cluster", + "multi": false, + "name": "rabbitmq_cluster", + "options": [], + "query": "label_values(rabbitmq_identity_info{namespace=\"$namespace\"}, rabbitmq_cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "15s", + "30s", + "1m", + "5m", + "10m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "RabbitMQ-Overview", + "uid": "Kn5xm-gZk", + "version": 20210308, + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ] +} \ No newline at end of file diff --git a/files/prom_rule_rmq_splitbrain.yaml b/files/prom_rule_rmq_splitbrain.yaml new file mode 100644 index 00000000..4a58740a --- /dev/null +++ b/files/prom_rule_rmq_splitbrain.yaml @@ -0,0 +1,10 @@ + - alert: RabbitMQ_split_brain + # detect if rabbitmq_queues is different between rabbitmq nodes + expr: count(count(rabbitmq_queues) by (job)) > 1 + for: 5m + labels: + severity: page + application: rabbitmq-server + annotations: + description: RabbitMQ split brain detected + summary: RabbitMQ split brain detected diff --git a/hooks/dashboards-relation-joined b/hooks/dashboards-relation-joined new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/dashboards-relation-joined @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/prometheus-rules-relation-created b/hooks/prometheus-rules-relation-created new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/prometheus-rules-relation-created @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/prometheus-rules-relation-joined b/hooks/prometheus-rules-relation-joined new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/prometheus-rules-relation-joined @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/rabbit_utils.py b/hooks/rabbit_utils.py index e808046a..27ce022d 100644 --- a/hooks/rabbit_utils.py +++ b/hooks/rabbit_utils.py @@ -655,6 +655,20 @@ def leave_cluster(): raise +def get_plugin_manager(): + """Find the path to the executable for managing plugins. + + :returns: Path to rabbitmq-plugins executable + :rtype: str + """ + # At version 3.8.2, only /sbin/rabbitmq-plugins can enable plugin correctly + if os.path.exists("/sbin/rabbitmq-plugins"): + return '/sbin/rabbitmq-plugins' + else: + return glob.glob( + '/usr/lib/rabbitmq/lib/rabbitmq_server-*/sbin/rabbitmq-plugins')[0] + + def _manage_plugin(plugin, action): os.environ['HOME'] = '/root' _rabbitmq_plugins = \ diff --git a/hooks/rabbitmq_server_relations.py b/hooks/rabbitmq_server_relations.py index 53095999..252944f4 100755 --- a/hooks/rabbitmq_server_relations.py +++ b/hooks/rabbitmq_server_relations.py @@ -79,6 +79,7 @@ from charmhelpers.core.hookenv import ( DEBUG, ERROR, INFO, + WARNING, leader_set, leader_get, relation_get, @@ -254,6 +255,58 @@ def update_clients(check_deferred_restarts=True): check_deferred_restarts=check_deferred_restarts) +@hooks.hook('dashboards-relation-joined') +def dashboards_relation_joined(relation_id=None, remote_unit=None): + """ + dashboards relation joined + send the dashboard json data via relation + """ + with open(os.path.join("files", "grafana-dashboard.json")) as f: + dashboard_str = f.read() + relation_set(relation_id, relation_settings={"dashboard": dashboard_str, + "name": "RabbitMQ-Overview"}) + + +@hooks.hook('prometheus-rules-relation-joined', + 'prometheus-rules-relation-created') +def prometheus_rules_joined(relation_id=None, remote_unit=None): + """ + prometheus rules relation joined + send the prometheus rules via relation + """ + with open(os.path.join("files", "prom_rule_rmq_splitbrain.yaml")) as f: + rule = f.read() + relation_set(relation_id, relation_settings={"groups": rule}) + + +@hooks.hook('scrape-relation-joined', 'scrape-relation-created') +def prometheus_scrape_joined(relation_id=None, remote_unit=None): + """ + scrape relation joined + enable prometheus plugin and open port + """ + err_msg = "rabbitmq-server needs to be >= 3.8 to support Prometheus plugin" + if cmp_pkgrevno('rabbitmq-server', '3.8.0') < 0: + log(err_msg, level=WARNING) + status_set("blocked", err_msg) + raise Exception(err_msg) + rabbit.enable_plugin(PROM_PLUGIN) + open_port(RMQ_MON_PORT) + relation_set(relation_id, relation_settings={"port": RMQ_MON_PORT}) + + +@hooks.hook('scrape-relation-broken') +def prometheus_scape_broken(): + """ + scrape relation broken + the relation has been completely removed + disable prometheus plugin and close port + """ + rabbit.disable_plugin(PROM_PLUGIN) + close_port(RMQ_MON_PORT) + log("scrape relation broken, disabled plugin and close port", level=INFO) + + @validate_amqp_config_tracker @hooks.hook('amqp-relation-changed') def amqp_changed(relation_id=None, remote_unit=None, @@ -664,6 +717,8 @@ def upgrade_charm(): MAN_PLUGIN = 'rabbitmq_management' +PROM_PLUGIN = 'rabbitmq_prometheus' +RMQ_MON_PORT = 15692 @hooks.hook('config-changed') diff --git a/hooks/scrape-relation-broken b/hooks/scrape-relation-broken new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/scrape-relation-broken @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/scrape-relation-created b/hooks/scrape-relation-created new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/scrape-relation-created @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/hooks/scrape-relation-joined b/hooks/scrape-relation-joined new file mode 120000 index 00000000..eecaaa74 --- /dev/null +++ b/hooks/scrape-relation-joined @@ -0,0 +1 @@ +rabbitmq_server_relations.py \ No newline at end of file diff --git a/metadata.yaml b/metadata.yaml index 4876e931..d6c4ac10 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -22,6 +22,12 @@ provides: nrpe-external-master: interface: nrpe-external-master scope: container + dashboards: + interface: grafana-dashboard + scrape: + interface: http + prometheus-rules: + interface: prometheus-rules requires: ha: interface: hacluster diff --git a/unit_tests/test_rabbit_utils.py b/unit_tests/test_rabbit_utils.py index 5ff5746f..aa7bd13f 100644 --- a/unit_tests/test_rabbit_utils.py +++ b/unit_tests/test_rabbit_utils.py @@ -893,6 +893,25 @@ class UtilsTests(CharmTestCase): mock_new_rabbitmq.return_value = True self.assertEqual(rabbit_utils.get_managment_port(), 15672) + @mock.patch('glob.glob') + @mock.patch('rabbit_utils.subprocess.check_call') + @mock.patch('os.path.exists') + def test_enable_management_plugin(self, mock_os_path, + mock_subprocess, + mock_glob): + mock_os_path.return_value = True + rabbitmq_plugins = '/sbin/rabbitmq-plugins' + rabbit_utils._manage_plugin("rabbitmq_prometheus", "enable") + mock_subprocess.assert_called_with([rabbitmq_plugins, + "enable", "rabbitmq_prometheus"]) + mock_os_path.return_value = False + rabbitmq_plugins = '/usr/lib/rabbitmq/lib/'\ + 'rabbitmq_server-3.8.2/sbin/rabbitmq-plugins' + mock_glob.return_value = [rabbitmq_plugins] + rabbit_utils._manage_plugin("rabbitmq_prometheus", "enable") + mock_subprocess.assert_called_with([rabbitmq_plugins, + "enable", "rabbitmq_prometheus"]) + @mock.patch('rabbit_utils.caching_cmp_pkgrevno') @mock.patch('rabbit_utils.relations_for_id') @mock.patch('rabbit_utils.subprocess')