Enable RabbitMQ Quorum Queues by default

This patch finalizes [1] and [2] topics and enables usage of quorum
queues by default along with adding release note and documentation
around the change.

We also convert previous quorum job to the classic HA queues
test, not to break an upgrade path for those who want to switch
to Quorum after OpenStack upgrade.

[1] https://review.opendev.org/q/topic:%22osa/messaging_improvements%22
[2] https://review.opendev.org/q/topic:%22osa/rmq-migrate%22

Depends-On: https://review.opendev.org/c/openstack/openstack-ansible/+/919902
Change-Id: I80e24343f28f712492d378186d0a0fa5832907e2

doc/source/admin/upgrades/major-upgrades.rst

@@ -155,6 +155,19 @@ Please review the contents of the playbook for more information.
     # openstack-ansible "${SCRIPTS_PATH}/upgrade-utilities/deploy-config-changes.yml"
 
 
+.. note::
+
+    With upgrade to 2024.1 (Caracal) release usage of RabbitMQ Quorum Queues
+    is enabled by default. Migration to usage of Quorum Queues results
+    in prolonged downtime for services during upgrade.
+
+    To reduce downtime you might want to set
+    ``oslomsg_rabbit_quorum_queues: false`` at this point and migrate to
+    Quorum Queues usage after OpenStack upgrade is done.
+
+    Please, check `RabbitMQ maintenance <https://docs.openstack.org/openstack-ansible/latest/admin/maintenance-tasks.html#migrate-between-ha-and-quorum-queues>`_
+    for more information about switching between Quourum and HA Queues.
+
 Upgrade hosts
 ~~~~~~~~~~~~~
 

inventory/group_vars/all/infra.yml

@@ -28,7 +28,6 @@ rabbitmq_servers: >-
 # If you do not want to use neither quorum queues nor classic mirror queues,
 # set rabbitmq_queue_replication to False
 rabbitmq_queue_replication: True
-oslomsg_rabbit_quorum_queues: False
 
 rabbitmq_policies:
   - name: "HA"
@@ -41,7 +40,11 @@ rabbitmq_policies:
     priority: 0
     tags:
       queue-version: 2
-    state: "{{ (oslomsg_rabbit_quorum_queues | default(True) or not rabbitmq_queue_replication) | ternary('present', 'absent') }}"
+    state: >-
+      {{
+        ((oslomsg_rabbit_quorum_queues | default(True) or not rabbitmq_queue_replication) and rabbitmq_install_method | default('') != 'distro'
+          ) | ternary('present', 'absent')
+      }}
 
 ## Galera options
 galera_client_package_state: "{{ package_state }}"

inventory/group_vars/all/oslo-messaging.yml

@@ -12,6 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Quorum Queues
+oslomsg_rabbit_quorum_queues: "{{ rabbitmq_queue_replication }}"
+# NOTE(noonedeadpunk): Disabled due to broken behaviour for metal setups
+#                      See https://bugs.launchpad.net/oslo.messaging/+bug/2065922
+oslomsg_rabbit_queue_manager: False
+
 # RPC
 oslomsg_rpc_transport: "{{ (groups[qdrouterd_host_group] | length > 0) | ternary('amqp', 'rabbit') }}"
 oslomsg_rpc_port: "{{ (groups[qdrouterd_host_group] | length > 0) | ternary(qdrouterd_port, rabbitmq_port) }}"

releasenotes/notes/quorum_queues_default-2f633cd0d5ee2037.yaml

@@ -0,0 +1,43 @@
+---
+prelude: >
+    RabbitMQ Quorum Queues are enabled by default along with other significant
+    improvements to oslo.messaging, like Queue Manager, Transient Queues to
+    use Quorum, usage of Stream Queues for fanout. You can check more details
+    for these options in `oslo.messaging release notes <https://docs.openstack.org/releasenotes/oslo.messaging/2024.1.html>`_
+features:
+  - |
+    Implemented new variables to control new oslo.messaging behaviour:
+
+    - oslomsg_rabbit_stream_fanout: Enabled if ``oslomsg_rabbit_quorum_queues``
+      is also set to `True`
+    - oslomsg_rabbit_transient_quorum_queues: Enabled if ``oslomsg_rabbit_stream_fanout``
+      is `True`
+    - oslomsg_rabbit_qos_prefetch_count: Must be set to a positive value if
+      ``oslomsg_rabbit_stream_fanout`` is `True`
+    - oslomsg_rabbit_queue_manager: Disabled by default. It is recommended to
+      enable only for containerized deployments.
+      Please check `oslo.messaging bug report <https://bugs.launchpad.net/oslo.messaging/+bug/2065922>`_
+      for more details of why it should not be used for metal deployments.
+
+    Each service also has corresponsive variables prefixed with service name, like
+    ``<service>_oslomsg_rabbit_stream_fanout`` to control them separately.
+upgrade:
+  - |
+    With enablement of RabbitMQ Quorum Queues by default, all vhosts will
+    be re-created and re-named without leading slash ('/'). For instance,
+    `/nova` vhost will be renamed to `nova`. This might affect configured
+    monitoring alert rules or statistics gathering.
+
+    Renaming process will also lead for prolonged downtime of services
+    during upgrade which will last from vhost renaming until service role
+    execution completition. This might be especially important for services
+    like Nova and Neutron where role runtime may take a while to cover all
+    hosts.
+
+    You can disable usage of Quorum Queues and use previous default of HA
+    Queues by defining ``oslomsg_rabbit_quorum_queues: False`` in your
+    user_variables.yml
+
+    Please check `RabbitMQ maintenance <https://docs.openstack.org/openstack-ansible/latest/admin/maintenance-tasks.html#migrate-between-ha-and-quorum-queues>`_
+    documentation for more infromation on how to switch between these 2 modes
+    with reduced downtime.

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -358,8 +358,8 @@ glance_default_store:
   type: cinder
 {% endif %}
 
-{% if 'quorum' in bootstrap_host_scenarios_expanded %}
-oslomsg_rabbit_quorum_queues: True
+{% if 'haqueues' in bootstrap_host_scenarios_expanded %}
+oslomsg_rabbit_quorum_queues: False
 {% endif %}
 
 # NOTE(jrosser) nested virt is not usable in general in opendev.org CI jobs,

zuul.d/jobs.yaml

@@ -437,12 +437,12 @@
     timeout: 10800
 
 - job:
-    name: openstack-ansible-deploy-aio_metal_quorum-ubuntu-jammy
+    name: openstack-ansible-deploy-aio_metal_haqueues-ubuntu-jammy
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-jammy
 
 - job:
-    name: openstack-ansible-upgrade-aio_metal_quorum-ubuntu-jammy
+    name: openstack-ansible-upgrade-aio_metal_haqueues-ubuntu-jammy
     parent: openstack-ansible-deploy-aio
     nodeset: ubuntu-jammy
     timeout: 10800

zuul.d/project-templates.yaml

@@ -189,7 +189,7 @@
         - openstack-ansible-deploy-aio_metal-ubuntu-jammy
         - openstack-ansible-deploy-aio_metal_tls-rockylinux-9
         - openstack-ansible-deploy-aio_metal_tls-ubuntu-jammy
-        - openstack-ansible-deploy-aio_metal_quorum-ubuntu-jammy
+        - openstack-ansible-deploy-aio_metal_haqueues-ubuntu-jammy
         - openstack-ansible-upgrade-aio_metal-rockylinux-9
         - openstack-ansible-upgrade_2023.1-aio_metal-rockylinux-9
         - openstack-ansible-upgrade-aio_metal-ubuntu-jammy
@@ -203,7 +203,7 @@
         - openstack-ansible-deploy-aio_metal-debian-bookworm
         - openstack-ansible-deploy-aio_metal-rockylinux-9
         - openstack-ansible-deploy-aio_metal-ubuntu-jammy
-        - openstack-ansible-deploy-aio_metal_quorum-ubuntu-jammy
+        - openstack-ansible-deploy-aio_metal_haqueues-ubuntu-jammy
         - openstack-ansible-deploy-aio_metal_tls-rockylinux-9
         - openstack-ansible-deploy-aio_metal_tls-ubuntu-jammy