From f902cd14fac7de4c4c9f7d019191268a6b4e9601 Mon Sep 17 00:00:00 2001
From: Pete Birley <pete@port.direct>
Date: Thu, 12 Apr 2018 22:32:58 -0500
Subject: [PATCH] RabbitMQ: recover from full cluster restart

This PS updates the RabbitMQ chart to name nodes via their hostnames
rather than IPs - allowing the cluster (and single nodes) to be
restarted without impact.

Additionally the rabbitmq managment interface is exposed and basic
helm tests have been added.

Change-Id: I84857d9f3697eaa8491aafaf6ee3b9d47dbf2191
---
 .../templates/bin/_rabbitmq-liveness.sh.tpl   |  2 +
 .../templates/bin/_rabbitmq-readiness.sh.tpl  |  2 +
 rabbitmq/templates/bin/_rabbitmq-test.sh.tpl  | 77 +++++++++++++++++++
 rabbitmq/templates/configmap-bin.yaml         |  2 +
 rabbitmq/templates/configmap-etc.yaml         | 19 ++---
 rabbitmq/templates/ingress-management.yaml    | 25 ++++++
 .../prometheus/exporter-deployment.yaml       | 28 +++----
 .../prometheus/exporter-service.yaml          |  4 +-
 rabbitmq/templates/pod-test.yaml              | 55 +++++++++++++
 rabbitmq/templates/service-discovery.yaml     | 39 ++++++++++
 .../templates/service-ingress-management.yaml | 25 ++++++
 rabbitmq/templates/service.yaml               |  2 +
 rabbitmq/templates/statefulset.yaml           | 46 +++++++----
 .../_to_rabbit_config.tpl}                    |  2 +-
 rabbitmq/values.yaml                          | 41 +++++++++-
 15 files changed, 320 insertions(+), 49 deletions(-)
 create mode 100644 rabbitmq/templates/bin/_rabbitmq-test.sh.tpl
 create mode 100644 rabbitmq/templates/ingress-management.yaml
 create mode 100644 rabbitmq/templates/pod-test.yaml
 create mode 100644 rabbitmq/templates/service-discovery.yaml
 create mode 100644 rabbitmq/templates/service-ingress-management.yaml
 rename rabbitmq/templates/{_helpers.tpl => utils/_to_rabbit_config.tpl} (95%)

diff --git a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl
index 4943ef54ef..2f30aa4373 100644
--- a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl
+++ b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl
@@ -16,4 +16,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+set -e
+
 exec rabbitmqctl status
diff --git a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl
index 4943ef54ef..2f30aa4373 100644
--- a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl
+++ b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl
@@ -16,4 +16,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
+set -e
+
 exec rabbitmqctl status
diff --git a/rabbitmq/templates/bin/_rabbitmq-test.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-test.sh.tpl
new file mode 100644
index 0000000000..04b2f0c451
--- /dev/null
+++ b/rabbitmq/templates/bin/_rabbitmq-test.sh.tpl
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -e
+
+# Extract connection details
+RABBIT_HOSTNAME=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $2}' \
+  | awk -F'[:/]' '{print $1}'`
+RABBIT_PORT=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $2}' \
+  | awk -F'[:/]' '{print $2}'`
+
+# Extract Admin User creadential
+RABBITMQ_ADMIN_USERNAME=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $1}' \
+  | awk -F'[//:]' '{print $4}'`
+RABBITMQ_ADMIN_PASSWORD=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $1}' \
+  | awk -F'[//:]' '{print $5}'`
+
+function rabbit_find_paritions () {
+  PARTITIONS=$(rabbitmqadmin \
+    --host="${RABBIT_HOSTNAME}" \
+    --port="${RABBIT_PORT}" \
+    --username="${RABBITMQ_ADMIN_USERNAME}" \
+    --password="${RABBITMQ_ADMIN_PASSWORD}" \
+    list nodes -f raw_json | \
+  python -c "import json,sys;
+obj=json.load(sys.stdin);
+for num, node in enumerate(obj):
+  print node['partitions'];")
+
+  for PARTITION in ${PARTITIONS}; do
+    if [[ $PARTITION != '[]' ]]; then
+      echo "Cluster partition found"
+      exit 1
+    fi
+  done
+  echo "No cluster partitions found"
+}
+# Check no nodes report cluster partitioning
+rabbit_find_paritions
+
+function rabbit_check_users_match () {
+  # Check users match on all nodes
+  NODES=$(rabbitmqadmin \
+    --host="${RABBIT_HOSTNAME}" \
+    --port="${RABBIT_PORT}" \
+    --username="${RABBITMQ_ADMIN_USERNAME}" \
+    --password="${RABBITMQ_ADMIN_PASSWORD}" \
+    list nodes -f bash)
+  USER_LIST=$(mktemp --directory)
+  for NODE in ${NODES}; do
+    rabbitmqadmin \
+      --host=${NODE#*@} \
+      --port="${RABBIT_PORT}" \
+      --username="${RABBITMQ_ADMIN_USERNAME}" \
+      --password="${RABBITMQ_ADMIN_PASSWORD}" \
+      list users -f bash > ${USER_LIST}/${NODE#*@}
+  done
+  cd ${USER_LIST}; diff -q --from-file $(ls ${USER_LIST})
+  echo "User lists match for all nodes"
+}
+# Check users match on all nodes
+rabbit_check_users_match
diff --git a/rabbitmq/templates/configmap-bin.yaml b/rabbitmq/templates/configmap-bin.yaml
index 600f523578..743e3077ce 100644
--- a/rabbitmq/templates/configmap-bin.yaml
+++ b/rabbitmq/templates/configmap-bin.yaml
@@ -22,6 +22,8 @@ kind: ConfigMap
 metadata:
   name: {{ printf "%s-%s" $envAll.Release.Name "rabbitmq-bin" | quote }}
 data:
+  rabbitmq-test.sh: |
+{{ tuple "bin/_rabbitmq-test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   rabbitmq-liveness.sh: |
 {{ tuple "bin/_rabbitmq-liveness.sh.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   rabbitmq-readiness.sh: |
diff --git a/rabbitmq/templates/configmap-etc.yaml b/rabbitmq/templates/configmap-etc.yaml
index 8f329b8f94..5eef45aff3 100644
--- a/rabbitmq/templates/configmap-etc.yaml
+++ b/rabbitmq/templates/configmap-etc.yaml
@@ -17,20 +17,17 @@ limitations under the License.
 {{- if .Values.manifests.configmap_etc }}
 {{- $envAll := . }}
 
-{{- if empty .Values.conf.rabbitmq.cluster_formation.k8s.service_name -}}
-{{- tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | set .Values.conf.rabbitmq.cluster_formation.k8s "service_name" | quote | trunc 0 -}}
-{{- end -}}
-{{- if empty .Values.conf.rabbitmq.cluster_formation.k8s.host -}}
-{{- print "kubernetes.default.svc." .Values.endpoints.cluster_domain_suffix | set .Values.conf.rabbitmq.cluster_formation.k8s "host" | quote | trunc 0 -}}
+{{- if empty $envAll.Values.conf.rabbitmq.cluster_formation.k8s.host -}}
+{{- print "kubernetes.default.svc." $envAll.Values.endpoints.cluster_domain_suffix | set $envAll.Values.conf.rabbitmq.cluster_formation.k8s "host" | quote | trunc 0 -}}
 {{- end -}}
 
-{{- print "0.0.0.0:" ( tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") | set .Values.conf.rabbitmq.listeners.tcp "1" | quote | trunc 0 -}}
+{{- print "0.0.0.0:" ( tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") | set $envAll.Values.conf.rabbitmq.listeners.tcp "1" | quote | trunc 0 -}}
 
-{{- if empty .Values.conf.rabbitmq.default_user -}}
-{{- set .Values.conf.rabbitmq "default_user" .Values.endpoints.oslo_messaging.auth.user.username | quote | trunc 0 -}}
+{{- if empty $envAll.Values.conf.rabbitmq.default_user -}}
+{{- set $envAll.Values.conf.rabbitmq "default_user" $envAll.Values.endpoints.oslo_messaging.auth.user.username | quote | trunc 0 -}}
 {{- end -}}
-{{- if empty .Values.conf.rabbitmq.default_pass -}}
-{{- set .Values.conf.rabbitmq "default_pass" .Values.endpoints.oslo_messaging.auth.user.password | quote | trunc 0 -}}
+{{- if empty $envAll.Values.conf.rabbitmq.default_pass -}}
+{{- set $envAll.Values.conf.rabbitmq "default_pass" $envAll.Values.endpoints.oslo_messaging.auth.user.password | quote | trunc 0 -}}
 {{- end -}}
 
 ---
@@ -42,5 +39,5 @@ data:
   enabled_plugins: |
 {{ tuple "etc/_enabled_plugins.tpl" . | include  "helm-toolkit.utils.template" | indent 4 }}
   rabbitmq.conf: |
-{{ include "rabbitmq.to_rabbit_config" .Values.conf.rabbitmq | indent 4 }}
+{{ include "rabbitmq.utils.to_rabbit_config" $envAll.Values.conf.rabbitmq | indent 4 }}
 {{ end }}
diff --git a/rabbitmq/templates/ingress-management.yaml b/rabbitmq/templates/ingress-management.yaml
new file mode 100644
index 0000000000..cdd2c925d8
--- /dev/null
+++ b/rabbitmq/templates/ingress-management.yaml
@@ -0,0 +1,25 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.ingress_management .Values.network.management.ingress.public }}
+{{- $envAll := . }}
+{{- if empty $envAll.Values.endpoints.oslo_messaging.hosts.public }}
+{{- $service_public_name := .Release.Name | trunc 12 }}
+{{- $_ := set $envAll.Values.endpoints.oslo_messaging.hosts "public" ( printf "%s-%s-%s" $service_public_name "mgr" ( $service_public_name | sha256sum | trunc 6 )) }}
+{{- end }}
+{{- $ingressOpts := dict "envAll" . "backendService" "management" "backendServiceType" "oslo_messaging" "backendPort" "http" -}}
+{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
+{{- end }}
diff --git a/rabbitmq/templates/monitoring/prometheus/exporter-deployment.yaml b/rabbitmq/templates/monitoring/prometheus/exporter-deployment.yaml
index 5767cd54a8..6cb2e27ba9 100644
--- a/rabbitmq/templates/monitoring/prometheus/exporter-deployment.yaml
+++ b/rabbitmq/templates/monitoring/prometheus/exporter-deployment.yaml
@@ -16,7 +16,7 @@ limitations under the License.
 
 {{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }}
 {{- $envAll := . }}
-{{- $dependencies := .Values.dependencies.static.prometheus_rabbitmq_exporter }}
+{{- $dependencies := $envAll.Values.dependencies.static.prometheus_rabbitmq_exporter }}
 
 {{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "rabbitmq-exporter"  }}
 {{ tuple $envAll $dependencies $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -26,41 +26,41 @@ kind: Deployment
 metadata:
   name: {{ $rcControllerName | quote }}
 spec:
-  replicas: {{ .Values.pod.replicas.prometheus_rabbitmq_exporter }}
+  replicas: {{ $envAll.Values.pod.replicas.prometheus_rabbitmq_exporter }}
 {{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }}
   template:
     metadata:
       labels:
 {{ tuple $envAll "prometheus_rabbitmq_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
-      namespace: {{ .Values.endpoints.prometheus_rabbitmq_exporter.namespace }}
+      namespace: {{ $envAll.Values.endpoints.prometheus_rabbitmq_exporter.namespace }}
     spec:
       serviceAccountName: {{ $rcControllerName | quote }}
       nodeSelector:
-        {{ .Values.labels.prometheus_rabbitmq_exporter.node_selector_key }}: {{ .Values.labels.prometheus_rabbitmq_exporter.node_selector_value }}
-      terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_rabbitmq_exporter.timeout | default "30" }}
+        {{ $envAll.Values.labels.prometheus_rabbitmq_exporter.node_selector_key }}: {{ $envAll.Values.labels.prometheus_rabbitmq_exporter.node_selector_value }}
+      terminationGracePeriodSeconds: {{ $envAll.Values.pod.lifecycle.termination_grace_period.prometheus_rabbitmq_exporter.timeout | default "30" }}
       initContainers:
 {{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
       containers:
         - name: rabbitmq-exporter
-          image: {{ .Values.images.tags.prometheus_rabbitmq_exporter }}
-          imagePullPolicy: {{ .Values.images.pull_policy }}
+          image: {{ $envAll.Values.images.tags.prometheus_rabbitmq_exporter }}
+          imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.prometheus_rabbitmq_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           ports:
             - name: metrics
-              containerPort: {{ .Values.network.prometheus_rabbitmq_exporter.port }}
+              containerPort: {{ $envAll.Values.network.prometheus_rabbitmq_exporter.port }}
           env:
           - name: RABBIT_URL
             value: http://{{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}:15672
           - name: RABBIT_USER
-            value: {{ .Values.endpoints.oslo_messaging.auth.user.username | quote }}
+            value: {{ $envAll.Values.endpoints.oslo_messaging.auth.user.username | quote }}
           - name: RABBIT_PASSWORD
-            value: {{ .Values.endpoints.oslo_messaging.auth.user.password | quote }}
+            value: {{ $envAll.Values.endpoints.oslo_messaging.auth.user.password | quote }}
           - name: RABBIT_CAPABILITIES
-            value: {{ tuple .Values.conf.prometheus_exporter.capabilities $envAll | include "helm-toolkit.utils.joinListWithComma" | quote }}
+            value: {{ tuple $envAll.Values.conf.prometheus_exporter.capabilities $envAll | include "helm-toolkit.utils.joinListWithComma" | quote }}
           - name: PUBLISH_PORT
-            value: {{ .Values.network.prometheus_rabbitmq_exporter.port | quote }}
+            value: {{ $envAll.Values.network.prometheus_rabbitmq_exporter.port | quote }}
           - name: LOG_LEVEL
-            value: {{ .Values.conf.prometheus_exporter.log_level | quote }}
+            value: {{ $envAll.Values.conf.prometheus_exporter.log_level | quote }}
           - name: SKIPVERIFY
-            value: {{ .Values.conf.prometheus_exporter.skipverify | quote }}
+            value: {{ $envAll.Values.conf.prometheus_exporter.skipverify | quote }}
 {{- end }}
diff --git a/rabbitmq/templates/monitoring/prometheus/exporter-service.yaml b/rabbitmq/templates/monitoring/prometheus/exporter-service.yaml
index fbcc21f227..f49a126748 100644
--- a/rabbitmq/templates/monitoring/prometheus/exporter-service.yaml
+++ b/rabbitmq/templates/monitoring/prometheus/exporter-service.yaml
@@ -25,13 +25,13 @@ metadata:
   labels:
 {{ tuple $envAll "prometheus_rabbitmq_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
   annotations:
-{{- if .Values.monitoring.prometheus.enabled }}
+{{- if $envAll.Values.monitoring.prometheus.enabled }}
 {{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }}
 {{- end }}
 spec:
   ports:
   - name: metrics
-    port: {{ .Values.network.prometheus_rabbitmq_exporter.port }}
+    port: {{ $envAll.Values.network.prometheus_rabbitmq_exporter.port }}
   selector:
 {{ tuple $envAll "prometheus_rabbitmq_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
 {{- end }}
diff --git a/rabbitmq/templates/pod-test.yaml b/rabbitmq/templates/pod-test.yaml
new file mode 100644
index 0000000000..b47678ba85
--- /dev/null
+++ b/rabbitmq/templates/pod-test.yaml
@@ -0,0 +1,55 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.pod_test }}
+{{- $envAll := . }}
+{{- $dependencies := $envAll.Values.dependencies.static.tests }}
+
+{{- $serviceAccountName := print .Release.Name "-test" }}
+{{ tuple $envAll $dependencies $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{.Release.Name}}-test"
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  serviceAccountName: {{ $serviceAccountName }}
+  nodeSelector:
+    {{ $envAll.Values.labels.test.node_selector_key }}: {{ $envAll.Values.labels.test.node_selector_value }}
+  restartPolicy: Never
+  initContainers:
+{{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
+  containers:
+    - name: {{.Release.Name}}-rabbitmq-test
+      image: {{ $envAll.Values.images.tags.scripted_test }}
+      env:
+        - name: RABBITMQ_ADMIN_CONNECTION
+          value: "{{ tuple "oslo_messaging" "internal" "user" "http" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" }}"
+      command:
+        - /tmp/rabbitmq-test.sh
+      volumeMounts:
+        - name: rabbitmq-bin
+          mountPath: /tmp/rabbitmq-test.sh
+          subPath: rabbitmq-test.sh
+          readOnly: true
+  volumes:
+    - name: rabbitmq-bin
+      configMap:
+        name: {{ printf "%s-%s" $envAll.Release.Name "rabbitmq-bin" | quote }}
+        defaultMode: 0555
+{{- end }}
diff --git a/rabbitmq/templates/service-discovery.yaml b/rabbitmq/templates/service-discovery.yaml
new file mode 100644
index 0000000000..54c16f27e7
--- /dev/null
+++ b/rabbitmq/templates/service-discovery.yaml
@@ -0,0 +1,39 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.service_discovery }}
+{{- $envAll := . }}
+{{- if empty $envAll.Values.endpoints.oslo_messaging.hosts.discovery }}
+{{- $service_discovery_name := .Release.Name | trunc 12 }}
+{{- $_ := set $envAll.Values.endpoints.oslo_messaging.hosts "discovery" ( printf "%s-%s-%s" $service_discovery_name "dsv" ( $service_discovery_name | sha256sum | trunc 6 )) }}
+{{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tuple "oslo_messaging" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+spec:
+  ports:
+  - port: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    name: amqp
+  - port: {{ add (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") 20000 }}
+    name: clustering
+  - port: {{ tuple "oslo_messaging" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    name: http
+  clusterIP: None
+  selector:
+{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }}
+{{ end }}
diff --git a/rabbitmq/templates/service-ingress-management.yaml b/rabbitmq/templates/service-ingress-management.yaml
new file mode 100644
index 0000000000..deca9b9901
--- /dev/null
+++ b/rabbitmq/templates/service-ingress-management.yaml
@@ -0,0 +1,25 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if and .Values.manifests.service_ingress_management .Values.network.management.ingress.public }}
+{{- $envAll := . }}
+{{- if empty $envAll.Values.endpoints.oslo_messaging.hosts.public }}
+{{- $service_public_name := .Release.Name | trunc 12 }}
+{{- $_ := set $envAll.Values.endpoints.oslo_messaging.hosts "public" ( printf "%s-%s-%s" $service_public_name "mgr" ( $service_public_name | sha256sum | trunc 6 )) }}
+{{- end }}
+{{- $serviceIngressOpts := dict "envAll" . "backendService" "management" "backendServiceType" "oslo_messaging" "backendPort" "http" -}}
+{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }}
+{{- end }}
diff --git a/rabbitmq/templates/service.yaml b/rabbitmq/templates/service.yaml
index e9ba424ceb..262226e4bd 100644
--- a/rabbitmq/templates/service.yaml
+++ b/rabbitmq/templates/service.yaml
@@ -25,6 +25,8 @@ spec:
   ports:
   - port: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     name: amqp
+  - port: {{ add (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") 20000 }}
+    name: clustering
   - port: {{ tuple "oslo_messaging" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     name: http
   selector:
diff --git a/rabbitmq/templates/statefulset.yaml b/rabbitmq/templates/statefulset.yaml
index c18cf41b42..e2935e9a70 100644
--- a/rabbitmq/templates/statefulset.yaml
+++ b/rabbitmq/templates/statefulset.yaml
@@ -16,7 +16,12 @@ limitations under the License.
 
 {{- if .Values.manifests.statefulset }}
 {{- $envAll := . }}
-{{- $dependencies := .Values.dependencies.static.rabbitmq }}
+{{- if empty $envAll.Values.endpoints.oslo_messaging.hosts.discovery }}
+{{- $service_discovery_name := .Release.Name | trunc 12 }}
+{{- $_ := set $envAll.Values.endpoints.oslo_messaging.hosts "discovery" ( printf "%s-%s-%s" $service_discovery_name "dsv" ( $service_discovery_name | sha256sum | trunc 6 )) }}
+{{- end }}
+
+{{- $dependencies := $envAll.Values.dependencies.static.rabbitmq }}
 
 {{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "rabbitmq"  }}
 {{ tuple $envAll $dependencies $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -58,8 +63,8 @@ kind: StatefulSet
 metadata:
   name: {{ $rcControllerName | quote }}
 spec:
-  serviceName: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
-  replicas: {{ .Values.pod.replicas.server }}
+  serviceName: {{ tuple "oslo_messaging" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+  replicas: {{ $envAll.Values.pod.replicas.server }}
   template:
     metadata:
       labels:
@@ -72,13 +77,13 @@ spec:
       affinity:
 {{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }}
       nodeSelector:
-        {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }}
+        {{ $envAll.Values.labels.server.node_selector_key }}: {{ $envAll.Values.labels.server.node_selector_value }}
       initContainers:
 {{ tuple $envAll $dependencies list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
-{{- if .Values.volume.chown_on_start }}
+{{- if $envAll.Values.volume.chown_on_start }}
         - name: rabbitmq-perms
-          image: {{ .Values.images.tags.rabbitmq }}
-          imagePullPolicy: {{ .Values.images.pull_policy }}
+          image: {{ $envAll.Values.images.tags.rabbitmq }}
+          imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
           securityContext:
             runAsUser: 0
 {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -93,7 +98,7 @@ spec:
 {{- end }}
       containers:
         - name: rabbitmq
-          image: {{ .Values.images.tags.rabbitmq }}
+          image: {{ $envAll.Values.images.tags.rabbitmq }}
 {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           command:
             - /tmp/rabbitmq-start.sh
@@ -104,19 +109,26 @@ spec:
             - name: amqp
               protocol: TCP
               containerPort: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+            - name: clustering
+              protocol: TCP
+              containerPort: {{ add (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") 20000 }}
           env:
-            - name: MY_POD_IP
+            - name: MY_POD_NAME
               valueFrom:
                 fieldRef:
-                  fieldPath: status.podIP
+                  fieldPath: metadata.name
             - name: RABBITMQ_USE_LONGNAME
               value: "true"
             - name: RABBITMQ_NODENAME
-              value: "rabbit@$(MY_POD_IP)"
+              value: "rabbit@$(MY_POD_NAME).{{ tuple "oslo_messaging" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}"
             - name: K8S_SERVICE_NAME
-              value: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }}
+              value: {{ tuple "oslo_messaging" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}
+            # NOTE(portdirect): We use the discovery fqdn here, as we resolve
+            # nodes via their pods hostname/nodename
+            - name: K8S_HOSTNAME_SUFFIX
+              value: ".{{ tuple "oslo_messaging" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}"
             - name: RABBITMQ_ERLANG_COOKIE
-              value: "{{ .Values.endpoints.oslo_messaging.auth.erlang_cookie }}"
+              value: "{{ $envAll.Values.endpoints.oslo_messaging.auth.erlang_cookie }}"
           readinessProbe:
             initialDelaySeconds: 10
             timeoutSeconds: 10
@@ -151,11 +163,11 @@ spec:
           configMap:
             name: {{ printf "%s-%s" $envAll.Release.Name "rabbitmq-etc" | quote }}
             defaultMode: 0444
-        {{- if not .Values.volume.enabled }}
+        {{- if not $envAll.Values.volume.enabled }}
         - name: rabbitmq-data
           emptyDir: {}
         {{- end }}
-{{- if .Values.volume.enabled }}
+{{- if $envAll.Values.volume.enabled }}
   volumeClaimTemplates:
     - metadata:
         name: rabbitmq-data
@@ -163,7 +175,7 @@ spec:
         accessModes: [ "ReadWriteOnce" ]
         resources:
           requests:
-            storage: {{ .Values.volume.size }}
-        storageClassName: {{ .Values.volume.class_name }}
+            storage: {{ $envAll.Values.volume.size }}
+        storageClassName: {{ $envAll.Values.volume.class_name }}
 {{- end }}
 {{ end }}
diff --git a/rabbitmq/templates/_helpers.tpl b/rabbitmq/templates/utils/_to_rabbit_config.tpl
similarity index 95%
rename from rabbitmq/templates/_helpers.tpl
rename to rabbitmq/templates/utils/_to_rabbit_config.tpl
index c62b49950d..fb90bd1728 100644
--- a/rabbitmq/templates/_helpers.tpl
+++ b/rabbitmq/templates/utils/_to_rabbit_config.tpl
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- define "rabbitmq.to_rabbit_config" -}}
+{{- define "rabbitmq.utils.to_rabbit_config" -}}
 {{- range $top_key, $top_value :=  . }}
 {{- if kindIs "map" $top_value -}}
 {{- range $second_key, $second_value :=  . }}
diff --git a/rabbitmq/values.yaml b/rabbitmq/values.yaml
index 64187d66cf..023c25e430 100644
--- a/rabbitmq/values.yaml
+++ b/rabbitmq/values.yaml
@@ -24,13 +24,17 @@ labels:
   prometheus_rabbitmq_exporter:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  test:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 images:
   tags:
     prometheus_rabbitmq_exporter: docker.io/kbudde/rabbitmq-exporter:v0.21.0
     prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:newton
-    rabbitmq: docker.io/rabbitmq:3.7.3
+    rabbitmq: docker.io/rabbitmq:3.7.4
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.0
+    scripted_test: docker.io/rabbitmq:3.7.4-management
   pull_policy: "IfNotPresent"
 
 pod:
@@ -94,15 +98,15 @@ conf:
   rabbitmq:
     listeners:
       tcp:
-        #NOTE(portdirect): This is always defined via the endpoints section.
+        # NOTE(portdirect): This is always defined via the endpoints section.
         1: null
     cluster_formation:
       peer_discovery_backend: rabbit_peer_discovery_k8s
       k8s:
-        address_type: ip
+        address_type: hostname
       node_cleanup:
         interval: "10"
-        only_log_warning: "false"
+        only_log_warning: "true"
     cluster_partition_handling: autoheal
     queue_master_locator: min-masters
     loopback_users.guest: "false"
@@ -121,6 +125,10 @@ dependencies:
           service: monitoring
     rabbitmq:
       jobs: null
+    tests:
+      services:
+        - endpoint: internal
+          service: oslo_messaging
 
 monitoring:
   prometheus:
@@ -129,6 +137,14 @@ monitoring:
       scrape: true
 
 network:
+  management:
+    ingress:
+      public: true
+      classes:
+        namespace: "nginx"
+        cluster: "nginx-cluster"
+      annotations:
+        nginx.ingress.kubernetes.io/rewrite-target: /
   prometheus_rabbitmq_exporter:
     port: 9095
 
@@ -161,15 +177,28 @@ endpoints:
         password: password
     hosts:
       default: rabbitmq
+      # NOTE(portdirect): If left empty, the release name sha suffixed with dsv
+      # will be used for to produce a unique hostname for clustering
+      # and discovery.
+      discovery: null
+      # NOTE(portdirect): the public host is only used to the management WUI
+      # If left empty, the release name sha suffixed with mgr, will be used to
+      # produce an unique hostname.
+      public: null
     host_fqdn_override:
       default: null
     path: /
     scheme: rabbit
     port:
+      clustering:
+        # NOTE(portdirect): the value for this port is driven by amqp+20000
+        # it should not be set manually.
+        default: null
       amqp:
         default: 5672
       http:
         default: 15672
+        public: 80
   prometheus_rabbitmq_exporter:
     namespace: null
     hosts:
@@ -193,10 +222,14 @@ volume:
 manifests:
   configmap_bin: true
   configmap_etc: true
+  ingress_management: true
+  pod_test: true
   monitoring:
     prometheus:
       configmap_bin: true
       deployment_exporter: true
       service_exporter: true
+  service_discovery: true
+  service_ingress_management: true
   service: true
   statefulset: true