From 61f84e3beb730468ba715753d2e51475d6b0b7d3 Mon Sep 17 00:00:00 2001
From: Matt Crees <mattc@stackhpc.com>
Date: Wed, 6 Dec 2023 16:04:52 +0000
Subject: [PATCH] Add precheck for RabbitMQ quorum queues

Adds a precheck to fail if non-quorum queues are found in RabbitMQ.

Currently excludes fanout and reply queues, pending support in
oslo.messaging [1].

[1]: https://review.opendev.org/c/openstack/oslo.messaging/+/888479

Closes-Bug: #2045887
Change-Id: Ibafdcd58618d97251a3405ef9332022d4d930e2b
---
 ansible/roles/rabbitmq/tasks/precheck.yml     | 30 +++++++++++++++++++
 ...ck-for-quorum-queues-ba10f58538959112.yaml |  9 ++++++
 tests/upgrade.sh                              |  6 ++--
 3 files changed, 43 insertions(+), 2 deletions(-)
 create mode 100644 releasenotes/notes/rabbitmq-add-precheck-for-quorum-queues-ba10f58538959112.yaml

diff --git a/ansible/roles/rabbitmq/tasks/precheck.yml b/ansible/roles/rabbitmq/tasks/precheck.yml
index ad663ce729..2f56307728 100644
--- a/ansible/roles/rabbitmq/tasks/precheck.yml
+++ b/ansible/roles/rabbitmq/tasks/precheck.yml
@@ -223,3 +223,33 @@
     - container_facts['rabbitmq'] is defined
     - om_enable_rabbitmq_high_availability | bool
   tags: rabbitmq-ha-precheck
+
+- block:
+    - name: List RabbitMQ queues
+      become: true
+      shell:
+        # TODO(mattcrees): remove ``| egrep -v '(fanout|reply)'`` once https://review.opendev.org/c/openstack/oslo.messaging/+/888479 is merged.
+        cmd: "{{ kolla_container_engine }} exec rabbitmq rabbitmqctl list_queues --silent name type | egrep -v '(fanout|reply)' | awk '{ print $NF }'"
+      register: rabbitmq_queues
+      changed_when: false
+      check_mode: false
+
+    - name: Filter RabbitMQ queue types
+      set_fact:
+        rabbitmq_queue_types: "{{ rabbitmq_queues.stdout | split | unique }}"
+
+    - name: Check if RabbitMQ quorum queues need to be configured
+      assert:
+        that: "{{ (rabbitmq_queue_types | length == 1) and (rabbitmq_queue_types | first == 'quorum') }}"
+        fail_msg: >
+          om_enable_rabbitmq_quorum_queues is True but non-quorum queues have been found.
+          Currently the procedure to migrate to quorum queues is manual.
+          Please follow the process described here: https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability.
+          Note that this process may take several hours on larger systems, and may cause a degredation in performance at large scale.
+          If you do not wish to enable this feature, set om_enable_rabbitmq_quorum_queues to False.
+
+  run_once: true
+  when:
+    - container_facts['rabbitmq'] is defined
+    - om_enable_rabbitmq_quorum_queues | bool
+  tags: rabbitmq-ha-precheck
diff --git a/releasenotes/notes/rabbitmq-add-precheck-for-quorum-queues-ba10f58538959112.yaml b/releasenotes/notes/rabbitmq-add-precheck-for-quorum-queues-ba10f58538959112.yaml
new file mode 100644
index 0000000000..cb2781fb9f
--- /dev/null
+++ b/releasenotes/notes/rabbitmq-add-precheck-for-quorum-queues-ba10f58538959112.yaml
@@ -0,0 +1,9 @@
+---
+fixes:
+  - |
+    A precheck has been added to catch when
+    ``om_enable_rabbitmq_quorum_queues`` is set to ``True``, but quorum queues
+    have not been configured on all appropriate queues. A manual migration is
+    required, see here for details:
+    https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability
+    `LP#2045887 <https://launchpad.net/bugs/2045887>`__
diff --git a/tests/upgrade.sh b/tests/upgrade.sh
index 78b945ce49..f897cabb61 100755
--- a/tests/upgrade.sh
+++ b/tests/upgrade.sh
@@ -12,11 +12,10 @@ function upgrade {
 
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
-    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks
+    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks --skip-tags rabbitmq-ha-precheck &> /tmp/logs/ansible/upgrade-prechecks
 
     # NOTE(mattcrees): As om_enable_rabbitmq_quorum_queues now defaults to
     # true in Bobcat, we need to perform a migration to durable queues.
-    # TODO(mattcrees): Remove these steps in Caracal.
     SERVICE_TAGS="heat,keystone,neutron,nova"
     if [[ $SCENARIO == "zun" ]] || [[ $SCENARIO == "cephadm" ]]; then
         SERVICE_TAGS+=",cinder"
@@ -44,6 +43,9 @@ function upgrade {
     kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade
     kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade
 
+    # Check that all appropriate RabbitMQ queues are now quorum queues.
+    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks --tags rabbitmq-ha-precheck &> /tmp/logs/ansible/rabbitmq-ha-precheck
+
     kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/upgrade-post-deploy
 
     kolla-ansible -i ${RAW_INVENTORY} -vvv validate-config &> /tmp/logs/ansible/validate-config