Followup on MariaDB handling fixes

This fixes issues reported by Mark:
- possible failure with 4-node cluster (however unlikely)
- failure to stop all nodes from progressing when conditions are
  not valid (due to: "any_errors_fatal: False")

Change-Id: Ib6995bf4c99202c9813859b3d9e2f420448f0445

ansible/roles/mariadb/handlers/main.yml

@@ -68,13 +68,14 @@
   when:
     - groups.mariadb_port_alive_True is defined
     - inventory_hostname in groups.mariadb_port_alive_True
-    - groups.mariadb_port_alive_True.index(inventory_hostname) % 3 == item
+    - groups.mariadb_port_alive_True.index(inventory_hostname) % 4 == item
     - kolla_action != "config"
   listen: restart mariadb
   loop:
     - 0
     - 1
     - 2
+    - 3
 
 - name: Start MariaDB on new nodes
   include_tasks: 'restart_services.yml'

ansible/roles/mariadb/tasks/lookup_cluster.yml

@@ -30,6 +30,15 @@
       group_by:
         key: mariadb_port_alive_{{ check_mariadb_port_liveness is success }}
 
+    - name: Fail on existing but stopped cluster
+      fail:
+        msg: MariaDB cluster exists but is stopped. Please start it using kolla-ansible mariadb_recovery
+      when:
+        # NOTE(yoctozepto): we allow single-node cluster to start
+        - groups['mariadb'] | length > 1
+        - mariadb_cluster_exists
+        - groups.mariadb_port_alive_True is not defined
+
     - block:
         - name: Check MariaDB service WSREP sync status
           become: true
@@ -48,18 +57,19 @@
         - name: Extract MariaDB service WSREP sync status
           set_fact:
             mariadb_sync_status: "{{ check_mariadb_sync_status.stdout.split('\t')[1] }}"
-
-        - name: Divide hosts by their MariaDB service WSREP sync status
-          group_by:
-            key: mariadb_sync_status_{{ mariadb_sync_status }}
-
-        - name: Fail when MariaDB service is not synced
-          fail:
-            msg: MariaDB service is not synced. Please wait for WSREP sync before proceeding.
-          when:
-            - groups.mariadb_sync_status_Synced is not defined or
-              inventory_hostname not in groups.mariadb_sync_status_Synced
       when:
         - groups.mariadb_port_alive_True is defined
         - inventory_hostname in groups.mariadb_port_alive_True
+
+    - name: Divide hosts by their MariaDB service WSREP sync status
+      group_by:
+        key: mariadb_sync_status_{{ mariadb_sync_status | default('NA') }}
+
+    - name: Fail when MariaDB services are not synced across the whole cluster
+      fail:
+        msg: MariaDB cluster is not synced. Please wait for WSREP sync before proceeding.
+      when:
+        - groups.mariadb_port_alive_True is defined
+        - groups.mariadb_sync_status_Synced is not defined or
+          groups.mariadb_port_alive_True | sort != groups.mariadb_sync_status_Synced | sort
   when: not mariadb_recover | default(False)