Matt Crees 423a42c65e Add precheck to fail if RabbitMQ HA needs configuring
Currently, the process of enabling RabbitMQ HA with the variable
``om_enable_rabbitmq_high_availbility`` requires some manual steps to
migrate from transient to mirrored queues. In preparation for setting
this variable to ``True`` by default, this adds a precheck that will
fail if a system is currently running non-mirrored queues and
``om_enable_rabbitmq_high_availbility`` is set to ``True``.

Includes a helpful message informing the operator of their choice.
Either follow the manual procedure to migrate the queues described in
the docs, or set ``om_enable_rabbitmq_high_availbility`` to ``False``.

The RabbitMQ HA section of the reference docs is updated to include
these instructions.

Change-Id: Ic5e64998bd01923162204f7bb289cc110187feec
(cherry picked from commit a5331d3208a0bcb4aada9b65acb71c51353cd9b9)
2023-04-20 08:27:28 +00:00

222 lines
7.4 KiB
YAML

---
- import_role:
name: service-precheck
vars:
service_precheck_services: "{{ rabbitmq_services }}"
service_name: "{{ project_name }}"
- name: Get container facts
become: true
kolla_container_facts:
container_engine: "{{ kolla_container_engine }}"
name:
- rabbitmq
- outward_rabbitmq
check_mode: false
register: container_facts
- name: Checking free port for RabbitMQ
wait_for:
host: "{{ api_interface_address }}"
port: "{{ rabbitmq_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['rabbitmq'] is not defined
- inventory_hostname in groups['rabbitmq']
- name: Checking free port for RabbitMQ Management
wait_for:
host: "{{ api_interface_address }}"
port: "{{ rabbitmq_management_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['rabbitmq'] is not defined
- inventory_hostname in groups['rabbitmq']
- name: Checking free port for RabbitMQ Cluster
wait_for:
host: "{{ api_interface_address }}"
port: "{{ rabbitmq_cluster_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['rabbitmq'] is not defined
- inventory_hostname in groups['rabbitmq']
- name: Checking free port for RabbitMQ EPMD
wait_for:
host: "{{ api_interface_address }}"
port: "{{ rabbitmq_epmd_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- container_facts['rabbitmq'] is not defined
- inventory_hostname in groups['rabbitmq']
- name: Check if all rabbit hostnames are resolvable
vars:
nss_database: "{{ 'ahostsv4' if api_address_family == 'ipv4' else 'ahostsv6' }}"
command: "getent {{ nss_database }} {{ hostvars[item].ansible_facts.hostname }}"
changed_when: false
check_mode: false
register: rabbitmq_hostnames
with_items: "{{ groups['rabbitmq'] }}"
- name: Check if each rabbit hostname resolves uniquely to the proper IP address
fail:
msg: Hostname has to resolve uniquely to the IP address of api_interface
with_subelements:
- "{{ rabbitmq_hostnames.results }}"
- stdout_lines
when:
- not item.1 is match('^'+('api' | kolla_address(item.0.item))+'\\b')
- name: Check if TLS certificate exists for RabbitMQ
vars:
cert: "{{ query('first_found', paths, errors='ignore') }}"
paths:
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/rabbitmq-cert.pem"
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
- "{{ kolla_certificates_dir }}/rabbitmq-cert.pem"
fail:
msg: No TLS certificate provided for RabbitMQ.
when:
- rabbitmq_enable_tls | bool
- cert | length == 0
- name: Check if TLS key exists for RabbitMQ
vars:
key: "{{ query('first_found', paths, errors='ignore') }}"
paths:
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/rabbitmq-key.pem"
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
- "{{ kolla_certificates_dir }}/rabbitmq-key.pem"
fail:
msg: No TLS key provided for RabbitMQ.
when:
- rabbitmq_enable_tls | bool
- key | length == 0
- name: Checking free port for outward RabbitMQ
wait_for:
host: "{{ api_interface_address }}"
port: "{{ outward_rabbitmq_port }}"
connect_timeout: 1
state: stopped
when:
- enable_outward_rabbitmq | bool
- inventory_hostname in groups['outward-rabbitmq']
- container_facts['outward_rabbitmq'] is not defined
- name: Checking free port for outward RabbitMQ Management
wait_for:
host: "{{ api_interface_address }}"
port: "{{ outward_rabbitmq_management_port }}"
connect_timeout: 1
state: stopped
when:
- enable_outward_rabbitmq | bool
- inventory_hostname in groups['outward-rabbitmq']
- container_facts['outward_rabbitmq'] is not defined
- name: Checking free port for outward RabbitMQ Cluster
wait_for:
host: "{{ api_interface_address }}"
port: "{{ outward_rabbitmq_cluster_port }}"
connect_timeout: 1
state: stopped
when:
- enable_outward_rabbitmq | bool
- inventory_hostname in groups['outward-rabbitmq']
- container_facts['outward_rabbitmq'] is not defined
- name: Checking free port for outward RabbitMQ EPMD
wait_for:
host: "{{ api_interface_address }}"
port: "{{ outward_rabbitmq_epmd_port }}"
connect_timeout: 1
state: stopped
when:
- enable_outward_rabbitmq | bool
- inventory_hostname in groups['outward-rabbitmq']
- container_facts['outward_rabbitmq'] is not defined
- name: Check if all outward rabbit hostnames are resolvable
vars:
nss_database: "{{ 'ahostsv4' if api_address_family == 'ipv4' else 'ahostsv6' }}"
command: "getent {{ nss_database }} {{ hostvars[item].ansible_facts.hostname }}"
changed_when: false
check_mode: false
register: outward_rabbitmq_hostnames
with_items: "{{ groups['outward-rabbitmq'] }}"
when:
- enable_outward_rabbitmq | bool
- name: Check if each rabbit hostname resolves uniquely to the proper IP address
fail:
msg: Hostname has to resolve uniquely to the IP address of api_interface
with_subelements:
- "{{ outward_rabbitmq_hostnames.results }}"
- stdout_lines
when:
- enable_outward_rabbitmq | bool
- not item.1 is match('^'+('api' | kolla_address(item.0.item))+'\\b')
- name: Check if TLS certificate exists for outward RabbitMQ
vars:
cert: "{{ query('first_found', paths, errors='ignore') }}"
paths:
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/outward_rabbitmq-cert.pem"
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-cert.pem"
- "{{ kolla_certificates_dir }}/outward_rabbitmq-cert.pem"
fail:
msg: No TLS certificate provided for outward RabbitMQ.
when:
- enable_outward_rabbitmq | bool
- rabbitmq_enable_tls | bool
- cert | length == 0
- name: Check if TLS key exists for outward RabbitMQ
vars:
key: "{{ query('first_found', paths, errors='ignore') }}"
paths:
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}/outward_rabbitmq-key.pem"
- "{{ kolla_certificates_dir }}/{{ inventory_hostname }}-key.pem"
- "{{ kolla_certificates_dir }}/outward_rabbitmq-key.pem"
fail:
msg: No TLS key provided for outward RabbitMQ.
when:
- enable_outward_rabbitmq | bool
- rabbitmq_enable_tls | bool
- key | length == 0
- block:
- name: List RabbitMQ policies
become: true
command: "{{ kolla_container_engine }} exec rabbitmq rabbitmqctl list_policies --silent"
register: rabbitmq_policies
changed_when: false
check_mode: false
- name: Check if RabbitMQ HA needs to be configured
assert:
that: "'ha-all' in rabbitmq_policies.stdout"
fail_msg: >
om_enable_rabbitmq_high_availability is True but no mirroring policy has been found.
Currently the procedure to migrate from transient non-mirrored queues to durable mirrored queues is manual.
Please follow the process described here: https://docs.openstack.org/kolla-ansible/latest/reference/message-queues/rabbitmq.html#high-availability.
Note that this process may take several hours on larger systems, and may cause a degredation in performance at large scale.
If you do not wish to enable this feature, set om_enable_rabbitmq_high_availability to False.
run_once: true
when:
- container_facts['rabbitmq'] is defined
- om_enable_rabbitmq_high_availability | bool
tags: rabbitmq-ha-precheck