From abdf99ad03d48c9aacf53d4234a4ff8f72b01e36 Mon Sep 17 00:00:00 2001 From: Arthur Luz de Avila Date: Wed, 2 Mar 2022 15:05:47 -0300 Subject: [PATCH] Update RabbitMQ probes The healthcheck used to liveness and readiness probes is considered intrusive and can produce false positives[0]. It's just returning that the pod is healthy if it's fully booted and whether the pod is waiting for a peer to appear to sync tables, it cannot succeed and eventually enters into a pod restart deadlock. The command is also deprecated and will be removed in future version. Updating the probes based on current recommenation from community[1]. Because of that, we change the liveness to check the rabbit connection to check if the rabbit is connected and running and also changed the readiness to ping the pod after boot to see if it's ready. Ref: [0] https://www.rabbitmq.com/monitoring.html#deprecations [1] https://www.rabbitmq.com/monitoring.html#health-checks Test plan: PASS: Close rabbitmq connection inside pod and verify if Liveness fail and rebooted the pod. PASS: Readiness check pod state and verify if can ping it to check if the pod is health after get up. PASS: Build on Debian Story: 2009892 Task: 44674 Signed-off-by: Arthur Luz de Avila Change-Id: Ib98df7e1135b7b5cee50d63a9ab06eb4a961cd4b --- .../centos/openstack-helm-infra.spec | 2 + .../patches/0023-Update-RabbitMQ-probes.patch | 70 +++++++++++++++++++ .../debian/deb_folder/patches/series | 1 + .../files/0023-Update-RabbitMQ-probes.patch | 70 +++++++++++++++++++ .../manifests/manifest.yaml | 2 +- 5 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 openstack-helm-infra/debian/deb_folder/patches/0023-Update-RabbitMQ-probes.patch create mode 100644 openstack-helm-infra/files/0023-Update-RabbitMQ-probes.patch diff --git a/openstack-helm-infra/centos/openstack-helm-infra.spec b/openstack-helm-infra/centos/openstack-helm-infra.spec index 23be46ee..5f44ec77 100644 --- a/openstack-helm-infra/centos/openstack-helm-infra.spec +++ b/openstack-helm-infra/centos/openstack-helm-infra.spec @@ -31,6 +31,7 @@ Patch19: 0019-Add-force_boot-command-to-rabbit-start-template.patch Patch20: 0020-Fix-tls-in-openstack-helm-infra.patch Patch21: 0021-Remove-mariadb-tls.patch Patch22: 0022-Remove-rabbitmq-tls.patch +Patch23: 0023-Update-RabbitMQ-probes.patch BuildRequires: helm BuildRequires: chartmuseum @@ -56,6 +57,7 @@ Openstack Helm Infra charts %patch20 -p1 %patch21 -p1 %patch22 -p1 +%patch23 -p1 %build # Host a server for the charts diff --git a/openstack-helm-infra/debian/deb_folder/patches/0023-Update-RabbitMQ-probes.patch b/openstack-helm-infra/debian/deb_folder/patches/0023-Update-RabbitMQ-probes.patch new file mode 100644 index 00000000..b148da1d --- /dev/null +++ b/openstack-helm-infra/debian/deb_folder/patches/0023-Update-RabbitMQ-probes.patch @@ -0,0 +1,70 @@ +From 3a76480c003dc6c1a522fba1c70278bad04930c2 Mon Sep 17 00:00:00 2001 +From: Roy Tang +Date: Fri, 13 Aug 2021 19:08:21 -0400 +Subject: [PATCH] Update RabbitMQ probes + +The current health check that is used for readiness and liveness +probes is considered intrusive and is prompt to produce false +positives[0]. The command is also deprecated and will be removed +in future version. Updating the probes based on current +recommenation from community[1]. + +Ref: +[0] https://www.rabbitmq.com/monitoring.html#deprecations +[1] https://www.rabbitmq.com/monitoring.html#health-checks + +Change-Id: I83750731150ff9a276f59e3c1288129581fceba5 +--- + rabbitmq/Chart.yaml | 2 +- + rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl | 3 +-- + rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl | 2 +- + releasenotes/notes/rabbitmq.yaml | 1 + + 4 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/rabbitmq/Chart.yaml b/rabbitmq/Chart.yaml +index 79b0daff..061ead2d 100644 +--- a/rabbitmq/Chart.yaml ++++ b/rabbitmq/Chart.yaml +@@ -15,6 +15,6 @@ apiVersion: v1 + appVersion: v3.7.26 + description: OpenStack-Helm RabbitMQ + name: rabbitmq +-version: 0.1.13 ++version: 0.1.14 + home: https://github.com/rabbitmq/rabbitmq-server + ... +diff --git a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl +index 943209aa..d07626b2 100644 +--- a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl ++++ b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl +@@ -19,6 +19,5 @@ set -e + if [ -f /tmp/rabbit-disable-liveness-probe ]; then + exit 0 + else +- timeout 5 bash -c "true &>/dev/null +Date: Fri, 13 Aug 2021 19:08:21 -0400 +Subject: [PATCH] Update RabbitMQ probes + +The current health check that is used for readiness and liveness +probes is considered intrusive and is prompt to produce false +positives[0]. The command is also deprecated and will be removed +in future version. Updating the probes based on current +recommenation from community[1]. + +Ref: +[0] https://www.rabbitmq.com/monitoring.html#deprecations +[1] https://www.rabbitmq.com/monitoring.html#health-checks + +Change-Id: I83750731150ff9a276f59e3c1288129581fceba5 +--- + rabbitmq/Chart.yaml | 2 +- + rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl | 3 +-- + rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl | 2 +- + releasenotes/notes/rabbitmq.yaml | 1 + + 4 files changed, 4 insertions(+), 4 deletions(-) + +diff --git a/rabbitmq/Chart.yaml b/rabbitmq/Chart.yaml +index 79b0daff..061ead2d 100644 +--- a/rabbitmq/Chart.yaml ++++ b/rabbitmq/Chart.yaml +@@ -15,6 +15,6 @@ apiVersion: v1 + appVersion: v3.7.26 + description: OpenStack-Helm RabbitMQ + name: rabbitmq +-version: 0.1.13 ++version: 0.1.14 + home: https://github.com/rabbitmq/rabbitmq-server + ... +diff --git a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl +index 943209aa..d07626b2 100644 +--- a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl ++++ b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl +@@ -19,6 +19,5 @@ set -e + if [ -f /tmp/rabbit-disable-liveness-probe ]; then + exit 0 + else +- timeout 5 bash -c "true &>/dev/null