Refactor MariaDB and RabbitMQ restart procedure

Ansible 2.14.3 introduced a change that broke the method used for
restarting MariaDB and RabbitMQ serially [1][2]. In
I57425680a4cdbf0daeb9b2cc35920f1b933aa4a8 we limited to 2.14.2 to work
around this. Ansible upstream claim this behaviour was unintentional,
and will not fix it.

This change moves to a different approach where we use separate plays
with a 'serial' keyword to execute the restart.

This change also removes the restriction on the maximum supported
version of 2.14.2 on ansible-core - any 2.14 release is now supported.

[1] 65366f663d
[2] https://github.com/ansible/ansible/issues/80848

Depends-On: https://review.opendev.org/c/openstack/kolla/+/884208

Change-Id: I5a12670d07077d24047aaff57ce8d33ccf7156ff

ansible/mariadb.yml

@@ -0,0 +1,72 @@
+---
+# For MariaDB we need to be careful about restarting services, to avoid losing quorum.
+- name: Apply role mariadb
+  gather_facts: false
+  hosts:
+    - mariadb
+    - '&enable_mariadb_True'
+  tags:
+    - mariadb
+  tasks:
+    - import_role:
+        name: mariadb
+
+- name: Restart mariadb services
+  gather_facts: false
+  hosts:
+    - mariadb_restart
+    - '&enable_mariadb_True'
+  # Restart in batches
+  serial: "33%"
+  tags:
+    - mariadb
+  tasks:
+    - import_role:
+        name: mariadb
+        tasks_from: restart_services.yml
+
+- name: Start mariadb services
+  gather_facts: false
+  hosts:
+    - mariadb_start
+    - '&enable_mariadb_True'
+  # Start in batches
+  serial: "33%"
+  tags:
+    - mariadb
+  tasks:
+    - import_role:
+        name: mariadb
+        tasks_from: restart_services.yml
+
+- name: Restart bootstrap mariadb service
+  gather_facts: false
+  hosts:
+    - mariadb_bootstrap_restart
+    - '&enable_mariadb_True'
+  tags:
+    - mariadb
+  tasks:
+    - import_role:
+        name: mariadb
+        tasks_from: restart_services.yml
+
+- name: Apply mariadb post-configuration
+  gather_facts: false
+  hosts:
+    - mariadb
+    - '&enable_mariadb_True'
+  tags:
+    - mariadb
+  tasks:
+    - name: Include mariadb post-deploy.yml
+      include_role:
+        name: mariadb
+        tasks_from: post-deploy.yml
+      when: kolla_action in ['deploy', 'reconfigure', 'upgrade']
+
+    - name: Include mariadb post-upgrade.yml
+      include_role:
+        name: mariadb
+        tasks_from: post-upgrade.yml
+      when: kolla_action == 'upgrade'

ansible/rabbitmq.yml

@@ -0,0 +1,115 @@
+---
+# For RabbitMQ we need to be careful about restarting services, to avoid losing quorum.
+- name: Apply role rabbitmq
+  gather_facts: false
+  hosts:
+    - rabbitmq
+    - '&enable_rabbitmq_True'
+  tags:
+    - rabbitmq
+  tasks:
+    - import_role:
+        name: rabbitmq
+      vars:
+        role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}'
+        role_rabbitmq_cluster_port: '{{ rabbitmq_cluster_port }}'
+        role_rabbitmq_epmd_port: '{{ rabbitmq_epmd_port }}'
+        role_rabbitmq_groups: rabbitmq
+        role_rabbitmq_management_port: '{{ rabbitmq_management_port }}'
+        role_rabbitmq_monitoring_password: '{{ rabbitmq_monitoring_password }}'
+        role_rabbitmq_monitoring_user: '{{ rabbitmq_monitoring_user }}'
+        role_rabbitmq_password: '{{ rabbitmq_password }}'
+        role_rabbitmq_port: '{{ rabbitmq_port }}'
+        role_rabbitmq_prometheus_port: '{{ rabbitmq_prometheus_port }}'
+        role_rabbitmq_user: '{{ rabbitmq_user }}'
+
+- name: Restart rabbitmq services
+  gather_facts: false
+  hosts:
+    - rabbitmq_restart
+    - '&enable_rabbitmq_True'
+  # Restart in batches
+  serial: "33%"
+  tags:
+    - rabbitmq
+  tasks:
+    - import_role:
+        name: rabbitmq
+        tasks_from: restart_services.yml
+      vars:
+        role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}'
+        role_rabbitmq_groups: rabbitmq
+
+- name: Apply rabbitmq post-configuration
+  gather_facts: false
+  hosts:
+    - rabbitmq
+    - '&enable_rabbitmq_True'
+  tags:
+    - rabbitmq
+  tasks:
+    - name: Include rabbitmq post-deploy.yml
+      include_role:
+        name: rabbitmq
+        tasks_from: post-deploy.yml
+      when: kolla_action in ['deploy', 'reconfigure']
+      vars:
+        role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}'
+        role_rabbitmq_groups: rabbitmq
+
+- name: Apply role rabbitmq (outward)
+  gather_facts: false
+  hosts:
+    - outward-rabbitmq
+    - '&enable_outward_rabbitmq_True'
+  tags:
+    - rabbitmq
+  tasks:
+    - import_role:
+        name: rabbitmq
+      vars:
+        project_name: outward_rabbitmq
+        role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}'
+        role_rabbitmq_cluster_port: '{{ outward_rabbitmq_cluster_port }}'
+        role_rabbitmq_epmd_port: '{{ outward_rabbitmq_epmd_port }}'
+        role_rabbitmq_groups: outward-rabbitmq
+        role_rabbitmq_management_port: '{{ outward_rabbitmq_management_port }}'
+        role_rabbitmq_password: '{{ outward_rabbitmq_password }}'
+        role_rabbitmq_port: '{{ outward_rabbitmq_port }}'
+        role_rabbitmq_prometheus_port: '{{ outward_rabbitmq_prometheus_port }}'
+        role_rabbitmq_user: '{{ outward_rabbitmq_user }}'
+
+- name: Restart rabbitmq (outward) services
+  gather_facts: false
+  hosts:
+    - outward_rabbitmq_restart
+    - '&enable_outward_rabbitmq_True'
+  # Restart in batches
+  serial: "33%"
+  tags:
+    - rabbitmq
+  tasks:
+    - import_role:
+        name: rabbitmq
+        tasks_from: restart_services.yml
+      vars:
+        project_name: outward_rabbitmq
+        role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}'
+        role_rabbitmq_groups: outward-rabbitmq
+
+- name: Apply rabbitmq (outward) post-configuration
+  gather_facts: false
+  hosts:
+    - outward-rabbitmq
+    - '&enable_outward_rabbitmq_True'
+  tags:
+    - rabbitmq
+  tasks:
+    - name: Include rabbitmq (outward) post-deploy.yml
+      include_role:
+        name: rabbitmq
+      when: kolla_action in ['deploy', 'reconfigure']
+      vars:
+        project_name: outward_rabbitmq
+        role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}'
+        role_rabbitmq_groups: outward-rabbitmq

ansible/roles/mariadb/handlers/main.yml

@@ -48,38 +48,29 @@
   no_log: true
   listen: Bootstrap MariaDB cluster
 
+- name: Ensure MariaDB is running normally on bootstrap host
+  group_by:
+    key: mariadb_bootstrap_restart
+  listen: Bootstrap MariaDB cluster
+
 - name: Restart MariaDB on existing cluster members
-  include_tasks: 'restart_services.yml'
+  group_by:
+    key: mariadb_restart
   when:
     - groups[mariadb_shard_group + '_port_alive_True'] is defined
     - inventory_hostname in groups[mariadb_shard_group + '_port_alive_True']
-    - groups[mariadb_shard_group + '_port_alive_True'].index(inventory_hostname) % 4 == item
     - kolla_action != "config"
   listen: restart mariadb
-  loop:
-    - 0
-    - 1
-    - 2
-    - 3
 
 - name: Start MariaDB on new nodes
-  include_tasks: 'restart_services.yml'
+  group_by:
+    key: mariadb_start
   when:
     - bootstrap_host is not defined or bootstrap_host != inventory_hostname
     - groups[mariadb_shard_group + '_port_alive_False'] is defined
     - inventory_hostname in groups[mariadb_shard_group + '_port_alive_False']
-    - groups[mariadb_shard_group + '_port_alive_False'].index(inventory_hostname) % 4 == item
     - kolla_action != "config"
   listen: restart mariadb
-  loop:
-    - 0
-    - 1
-    - 2
-    - 3
-
-- name: Ensure MariaDB is running normally on bootstrap host
-  include_tasks: 'restart_services.yml'
-  listen: Bootstrap MariaDB cluster
 
 - name: Restart mariadb-clustercheck container
   vars:

ansible/roles/mariadb/tasks/deploy.yml

@@ -4,10 +4,3 @@
 - import_tasks: check-containers.yml
 
 - import_tasks: bootstrap.yml
-
-- name: Flush handlers
-  meta: flush_handlers
-
-- import_tasks: register.yml
-
-- import_tasks: check.yml

ansible/roles/mariadb/tasks/post-deploy.yml

@@ -0,0 +1,4 @@
+---
+- import_tasks: register.yml
+
+- import_tasks: check.yml

ansible/roles/mariadb/tasks/post-upgrade.yml

@@ -0,0 +1,24 @@
+---
+- name: Run upgrade in MariaDB container
+  vars:
+    service_name: "mariadb"
+    service: "{{ mariadb_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "start_container"
+    common_options: "{{ docker_common_options }}"
+    detach: False
+    dimensions: "{{ service.dimensions }}"
+    environment:
+      KOLLA_UPGRADE:
+      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
+      DB_HOST: "{{ api_interface_address }}"
+      DB_PORT: "{{ mariadb_port }}"
+      DB_ROOT_PASSWORD: "{{ database_password }}"
+    image: "{{ service.image }}"
+    labels:
+      UPGRADE:
+    name: "upgrade_mariadb"
+    restart_policy: no
+    volumes: "{{ service.volumes }}"
+  no_log: true

ansible/roles/mariadb/tasks/upgrade.yml

@@ -1,26 +1,2 @@
 ---
 - import_tasks: deploy.yml
-
-- name: Run upgrade in MariaDB container
-  vars:
-    service_name: "mariadb"
-    service: "{{ mariadb_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "start_container"
-    common_options: "{{ docker_common_options }}"
-    detach: False
-    dimensions: "{{ service.dimensions }}"
-    environment:
-      KOLLA_UPGRADE:
-      KOLLA_CONFIG_STRATEGY: "{{ config_strategy }}"
-      DB_HOST: "{{ api_interface_address }}"
-      DB_PORT: "{{ mariadb_port }}"
-      DB_ROOT_PASSWORD: "{{ database_password }}"
-    image: "{{ service.image }}"
-    labels:
-      UPGRADE:
-    name: "upgrade_mariadb"
-    restart_policy: no
-    volumes: "{{ service.volumes }}"
-  no_log: true

ansible/roles/prechecks/vars/main.yml

@@ -2,7 +2,7 @@
 docker_version_min: '18.09'
 docker_py_version_min: '3.4.1'
 ansible_version_min: '2.13'
-ansible_version_max: '2.14.2'
+ansible_version_max: '2.14'
 
 # Top level keys should match ansible_facts.distribution.
 # These map to lists of supported releases (ansible_facts.distribution_release) or

ansible/roles/rabbitmq/handlers/main.yml

@@ -1,26 +1,6 @@
 ---
-# NOTE(mgoddard): These tasks perform a 'full stop upgrade', which is necessary when moving between
-# major releases. In future kolla-ansible releases we may be able to change this to a rolling
-# restart. For info on this process see https://www.rabbitmq.com/upgrade.html
-
-- name: Restart first rabbitmq container
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  include_tasks: 'restart_services.yml'
+- name: Restart rabbitmq container
+  group_by:
+    key: "{{ project_name }}_restart"
   when:
     - kolla_action != "config"
-    - inventory_hostname == groups[service.group] | first
-  listen: Restart rabbitmq container
-
-- name: Restart remaining rabbitmq containers
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  include_tasks: 'restart_services.yml'
-  when:
-    - kolla_action != "config"
-    - inventory_hostname == item
-    - inventory_hostname != groups[service.group] | first
-  loop: "{{ groups[service.group] }}"
-  listen: Restart rabbitmq container

ansible/roles/rabbitmq/tasks/deploy.yml

@@ -8,8 +8,3 @@
 - import_tasks: check-containers.yml
 
 - import_tasks: bootstrap.yml
-
-- name: Flush handlers
-  meta: flush_handlers
-
-- import_tasks: feature-flags.yml

ansible/roles/rabbitmq/tasks/post-deploy.yml

@@ -0,0 +1,2 @@
+---
+- import_tasks: feature-flags.yml

ansible/roles/rabbitmq/tasks/upgrade.yml

@@ -8,6 +8,3 @@
 - import_tasks: feature-flags.yml
 
 - import_tasks: check-containers.yml
-
-- name: Flush handlers
-  meta: flush_handlers

ansible/site.yml

@@ -380,14 +380,9 @@
     - { role: redis,
         tags: redis }
 
-- name: Apply role mariadb
-  gather_facts: false
-  hosts:
-    - mariadb
-    - '&enable_mariadb_True'
-  roles:
-    - { role: mariadb,
-        tags: mariadb }
+# MariaDB deployment is more complicated than other services, so is covered in
+# its own playbook.
+- import_playbook: mariadb.yml
 
 - name: Apply role memcached
   gather_facts: false
@@ -440,44 +435,7 @@
     - { role: multipathd,
         tags: multipathd }
 
-- name: Apply role rabbitmq
-  gather_facts: false
-  hosts:
-    - rabbitmq
-    - '&enable_rabbitmq_True'
-  roles:
-    - { role: rabbitmq,
-        tags: rabbitmq,
-        role_rabbitmq_cluster_cookie: '{{ rabbitmq_cluster_cookie }}',
-        role_rabbitmq_cluster_port: '{{ rabbitmq_cluster_port }}',
-        role_rabbitmq_epmd_port: '{{ rabbitmq_epmd_port }}',
-        role_rabbitmq_groups: rabbitmq,
-        role_rabbitmq_management_port: '{{ rabbitmq_management_port }}',
-        role_rabbitmq_monitoring_password: '{{ rabbitmq_monitoring_password }}',
-        role_rabbitmq_monitoring_user: '{{ rabbitmq_monitoring_user }}',
-        role_rabbitmq_password: '{{ rabbitmq_password }}',
-        role_rabbitmq_port: '{{ rabbitmq_port }}',
-        role_rabbitmq_prometheus_port: '{{ rabbitmq_prometheus_port }}',
-        role_rabbitmq_user: '{{ rabbitmq_user }}' }
-
-- name: Apply role rabbitmq (outward)
-  gather_facts: false
-  hosts:
-    - outward-rabbitmq
-    - '&enable_outward_rabbitmq_True'
-  roles:
-    - { role: rabbitmq,
-        tags: rabbitmq,
-        project_name: outward_rabbitmq,
-        role_rabbitmq_cluster_cookie: '{{ outward_rabbitmq_cluster_cookie }}',
-        role_rabbitmq_cluster_port: '{{ outward_rabbitmq_cluster_port }}',
-        role_rabbitmq_epmd_port: '{{ outward_rabbitmq_epmd_port }}',
-        role_rabbitmq_groups: outward-rabbitmq,
-        role_rabbitmq_management_port: '{{ outward_rabbitmq_management_port }}',
-        role_rabbitmq_password: '{{ outward_rabbitmq_password }}',
-        role_rabbitmq_port: '{{ outward_rabbitmq_port }}',
-        role_rabbitmq_prometheus_port: '{{ outward_rabbitmq_prometheus_port }}',
-        role_rabbitmq_user: '{{ outward_rabbitmq_user }}' }
+- import_playbook: rabbitmq.yml
 
 - name: Apply role etcd
   gather_facts: false

doc/source/user/quickstart.rst

@@ -90,12 +90,10 @@ Install dependencies for the virtual environment
       pip install -U pip
 
 #. Install `Ansible <http://www.ansible.com>`__. Kolla Ansible requires at least
-   Ansible ``6`` and supports up to ``7``. Ansible-core must not be greater
-   than 2.14.2 due to a regression.
+   Ansible ``6`` and supports up to ``7``.
 
    .. code-block:: console
 
-      pip install 'ansible-core>=2.13,<=2.14.2'
       pip install 'ansible>=6,<8'
 
 

doc/source/user/virtual-environments.rst

@@ -26,7 +26,6 @@ python virtual environment on the Ansible control host. For example:
    source /path/to/venv/bin/activate
    pip install -U pip
    pip install kolla-ansible
-   pip install 'ansible-core>=2.13,<=2.14.2'
    pip install 'ansible>=6,<8'
    deactivate
 

releasenotes/notes/mariadb-restart-refactor-529502bf5d12b0f4.yaml

@@ -0,0 +1,10 @@
+---
+upgrade:
+  - |
+    Removes the restriction on the maximum supported version of 2.14.2 for
+    ``ansible-core``. Any 2.14 series release is now supported.
+other:
+  - |
+    Refactors the MariaDB and RabbitMQ restart procedures to be compatible with
+    Ansible 2.14.3+.  See `Ansible issue 80848
+    <https://github.com/ansible/ansible/issues/80848>`__ for details.

tests/run.yml

@@ -245,7 +245,7 @@
     - name: install kolla-ansible and dependencies
       vars:
         ansible_core_version_min: "==2.13.*"
-        ansible_core_version_max: "==2.14.2"
+        ansible_core_version_max: "==2.14.*"
         # Test latest ansible version on Ubuntu, minimum supported on others.
         ansible_core_version_constraint: >-
           {{ ansible_core_version_min if is_upgrade or base_distro != 'ubuntu' else ansible_core_version_max }}