RabbitMQ handler refactored to restart services in serial

Adapted for Ussuri and below due to no RMQ TLS (different notifies).

Change-Id: I1ff4cbdf3f60cb7fd5fe5d3c5d498e05fe2df79a
Closes-Bug: #1904702
(cherry picked from commit 4cc4ba59da)

ansible/roles/rabbitmq/handlers/main.yml

@@ -1,45 +1,10 @@
 ---
-- name: Restart rabbitmq container (first node)
+- name: Restart rabbitmq container
   vars:
     service_name: "rabbitmq"
     service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    environment: "{{ service.environment }}"
-    dimensions: "{{ service.dimensions }}"
+  include_tasks: 'restart_services.yml'
   when:
     - kolla_action != "config"
-    - inventory_hostname == groups[service.group]|first
-  notify:
-    - Waiting for rabbitmq to start on first node
-
-- name: Waiting for rabbitmq to start on first node
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  command: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}"
-  when:
-    - inventory_hostname == groups[service.group]|first
-
-- name: Restart rabbitmq container (rest of nodes)
-  vars:
-    service_name: "rabbitmq"
-    service: "{{ rabbitmq_services[service_name] }}"
-  become: true
-  kolla_docker:
-    action: "recreate_or_restart_container"
-    common_options: "{{ docker_common_options }}"
-    name: "{{ service.container_name }}"
-    image: "{{ service.image }}"
-    volumes: "{{ service.volumes }}"
-    environment: "{{ service.environment }}"
-    dimensions: "{{ service.dimensions }}"
-  when:
-    - kolla_action != "config"
-    - inventory_hostname != groups[service.group]|first
+    - inventory_hostname == item
+  loop: "{{ groups[service.group] }}"

ansible/roles/rabbitmq/tasks/check-containers.yml

@@ -14,5 +14,4 @@
     - item.value.enabled | bool
   with_dict: "{{ rabbitmq_services }}"
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container

ansible/roles/rabbitmq/tasks/config.yml

@@ -23,8 +23,7 @@
     - item.value.enabled | bool
   with_dict: "{{ rabbitmq_services }}"
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
 
 - name: Copying over rabbitmq-env.conf
   become: true
@@ -42,8 +41,7 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
 
 
 - name: Copying over rabbitmq.conf
@@ -62,8 +60,7 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
 
 - name: Copying over erl_inetrc
   become: true
@@ -81,8 +78,7 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container
 
 - name: Copying over definitions.json
   become: true
@@ -100,5 +96,4 @@
     - inventory_hostname in groups[service.group]
     - service.enabled | bool
   notify:
-    - Restart rabbitmq container (first node)
-    - Restart rabbitmq container (rest of nodes)
+    - Restart rabbitmq container

ansible/roles/rabbitmq/tasks/restart_services.yml

@@ -0,0 +1,21 @@
+---
+- name: Restart rabbitmq container
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+    environment: "{{ service.environment }}"
+    dimensions: "{{ service.dimensions }}"
+
+- name: Waiting for rabbitmq to start
+  vars:
+    service_name: "rabbitmq"
+    service: "{{ rabbitmq_services[service_name] }}"
+  become: true
+  command: "docker exec {{ service.container_name }} rabbitmqctl wait {{ rabbitmq_pid_file }}"

releasenotes/notes/bug-1904702-7451dd8c4caa309b.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    RabbitMQ services are now restarted serially to avoid a split brain.
+    `LP#1904702 <https://launchpad.net/bugs/1904702>`__