Notify peers bootstrap-uuid during upgrade-charm hook

Since 17.02 the charm relies on the existence of bootstrap-uuid to
determine if pxc is bootstraped. The upgrade-charm hook handler uses
leader_node_is_ready() which calls cluster_ready(), this function will
evaluate to False when bootstrap-uuid is not defined

This patch drops leader_node_is_ready() from the upgrade-charm hook to
simply rely on is_leader(), for the non-leader unit
leader_get('bootstrap-uuid') and then notify_bootstrapped() if it's
set. For cases where the non-leader units are upgraded before
notify_bootstrapped() is called as part of the leader-settings-changed
hooks.

Change-Id: I621e23d7666920b91614491927711a85370310ad
Closes-Bug: 1674467

hooks/percona_hooks.py

@@ -27,6 +27,7 @@ from charmhelpers.core.hookenv import (
     charm_name,
     leader_get,
     open_port,
+    status_set,
 )
 from charmhelpers.core.host import (
     service_restart,
@@ -101,6 +102,8 @@ from percona_utils import (
     sst_password,
     root_password,
     pxc_installed,
+    update_bootstrap_uuid,
+    LeaderNoBootstrapUUIDError,
 )
 
 
@@ -261,15 +264,28 @@ def update_shared_db_rels():
 @harden()
 def upgrade():
 
-    if leader_node_is_ready():
-        # If this is the leader but we have not yet broadcast the cluster uuid
-        # then do so now.
+    if is_leader():
+        if is_unit_paused_set():
+            log('Unit is paused, skiping upgrade', level=INFO)
+            return
+
+        # broadcast the bootstrap-uuid
         wsrep_ready = get_wsrep_value('wsrep_ready') or ""
         if wsrep_ready.lower() in ['on', 'ready']:
             cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
             if cluster_state_uuid:
                 mark_seeded()
                 notify_bootstrapped(cluster_uuid=cluster_state_uuid)
+    else:
+        # Ensure all the peers have the bootstrap-uuid attribute set
+        # as this is all happening during the upgrade-charm hook is reasonable
+        # to expect the cluster is running.
+
+        # Wait until the leader has set the
+        try:
+            update_bootstrap_uuid()
+        except LeaderNoBootstrapUUIDError:
+            status_set('waiting', "Waiting for bootstrap-uuid set by leader")
 
     config_changed()
 
@@ -697,6 +713,14 @@ def leader_settings_changed():
     install_percona_xtradb_cluster()
     # Notify any changes to data in leader storage
     update_shared_db_rels()
+    log('leader-settings-changed', level='DEBUG')
+    try:
+        update_bootstrap_uuid()
+    except LeaderNoBootstrapUUIDError:
+        # until the bootstrap-uuid attribute is not replicated cluster_ready()
+        # will evaluate to False, so it is necessary to feed back this info
+        # to the user.
+        status_set('waiting', "Waiting for bootstrap-uuid set by leader")
 
 
 @hooks.hook('nrpe-external-master-relation-joined',