Add retry logic to mysql_upgrade

This change adds a retry to the mysql_upgrade command when performing
an upgrade. This is required in circumstances when the wsrep service
is not ready for application use. This is a common issue when restarting
the mysql service after package installation.

Example Error:
  Got error: 1047: WSREP has not yet prepared node for application
  use when selecting the database

Change-Id: Icd859531b6a6f1e04ecd1214eb7c401fa0fe8048
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

tasks/galera_running_check.yml

@@ -25,6 +25,21 @@
     - galera-cluster-state-check
     - galera-bootstrap
 
+- name: Check that WSREP is ready
+  shell: "/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf extended-status | egrep '(wsrep_ready|wsrep_evs_state)'"
+  changed_when: >
+    _mysql_ready.rc != 0
+  register: _mysql_ready
+  when: _mysql_running.rc == 0
+  until:
+    - _mysql_ready.rc == 0
+    - (_mysql_ready.stdout).find("ON") != -1
+  retries: "{{ num_retries }}"
+  delay: "{{ wait_delay }}"
+  tags:
+    - galera-cluster-state-check
+    - galera-bootstrap
+
 - name: Set running state fact
   set_fact:
     mysql_running: "{{ _mysql_running }}"

tasks/galera_setup.yml

@@ -41,5 +41,9 @@
       password: "{{ galera_monitoring_user_password }}"
       priv: "*.*:USAGE"
       state: present
+  register: galera_users
+  until: galera_users | success
+  retries: 10
+  delay: 3
   tags:
     - galera-user-create

tasks/galera_upgrade_post.yml

@@ -35,11 +35,15 @@
 
 - include: galera_running_check.yml
   vars:
-    num_retries: 1
+    num_retries: 10
     wait_delay: 3
 
 - name: Run MySQL Upgrade
   command: "/usr/bin/mysql_upgrade"
+  register: galera_mysql_upgrade
+  until: galera_mysql_upgrade | success
+  retries: 3
+  delay: 10
   when:
     - galera_upgrade | bool
   tags: