New member_ready state in peer relation

This change adds a new 'member_ready' state to the peer relation.
This purpose of this is to indicate that a unit has rendered its
configuration and started its services. This is distinct from the
existing 'ready' indicator. 'ready' indicates that the unit can
render its config and start services and ensures that all units are
present prior to starting services. This seems to safe guard
against units starting to early and forming single node clusters or
small clusters with a sub-set of units. The 'member_ready' flag
is later in the process and if it is set it shows that this unit
has joined the cluster and can be referenceed explicitly in any
resource configuration.

Change-Id: I80d42a628a3fe51bc6f8d02610031afd9386d7a4

hooks/hooks.py

@@ -459,6 +459,11 @@ def ha_relation_changed():
     for rel_id in relation_ids('ha'):
         relation_set(relation_id=rel_id, clustered="yes")
 
+    # Inform peers that local configuration is complete and this member
+    # is ready
+    for rel_id in relation_ids('hanode'):
+        relation_set(relation_id=rel_id, member_ready=True)
+
 
 @hooks.hook()
 def stop():

hooks/utils.py

@@ -458,7 +458,14 @@ def get_ha_nodes():
     return ha_nodes
 
 
-def get_cluster_nodes():
+def get_node_flags(flag):
+    """Nodes which have advertised the given flag.
+
+    :param flag: Flag to check peers relation data for.
+    :type flag: str
+    :returns: List of IPs of nodes that are ready to join the cluster
+    :rtype: List
+    """
     hosts = []
     if config('prefer-ipv6'):
         hosts.append(get_ipv6_addr())
@@ -467,13 +474,33 @@ def get_cluster_nodes():
 
     for relid in relation_ids('hanode'):
         for unit in related_units(relid):
-            if relation_get('ready', rid=relid, unit=unit):
-                hosts.append(relation_get('private-address', unit, relid))
+            if relation_get(flag, rid=relid, unit=unit):
+                hosts.append(relation_get('private-address',
+                                          rid=relid,
+                                          unit=unit))
 
     hosts.sort()
     return hosts
 
 
+def get_cluster_nodes():
+    """Nodes which have advertised that they are ready to join the cluster.
+
+    :returns: List of IPs of nodes that are ready to join the cluster
+    :rtype: List
+    """
+    return get_node_flags('ready')
+
+
+def get_member_ready_nodes():
+    """List of nodes which have advertised that they have joined the cluster.
+
+    :returns: List of IPs of nodes that have joined thcluster.
+    :rtype: List
+    """
+    return get_node_flags('member_ready')
+
+
 def parse_data(relid, unit, key):
     """Helper to detect and parse json or ast based relation data"""
     _key = 'json_{}'.format(key)

unit_tests/test_hacluster_utils.py

@@ -629,3 +629,49 @@ class UtilsTestCase(unittest.TestCase):
         config.side_effect = lambda x: cfg.get(x)
         with self.assertRaises(ValueError):
             utils.configure_maas_stonith_resource('node1')
+
+    @mock.patch.object(utils, 'unit_get')
+    @mock.patch.object(utils, 'get_ipv6_addr')
+    @mock.patch.object(utils, 'relation_get')
+    @mock.patch.object(utils, 'related_units')
+    @mock.patch.object(utils, 'relation_ids')
+    @mock.patch.object(utils, 'config')
+    def test_get_node_flags(self, config, relation_ids, related_units,
+                            relation_get, get_ipv6_addr, unit_get):
+        cfg = {}
+        config.side_effect = lambda x: cfg.get(x)
+        unit_get.return_value = '10.0.0.41'
+        relation_ids.return_value = ['relid1']
+        related_units.return_value = ['unit1', 'unit2', 'unit3']
+        rdata = {
+            'unit1': {
+                'random_flag': True,
+                'private-address': '10.0.0.10'},
+            'unit2': {
+                'random_flag': True,
+                'private-address': '10.0.0.34'},
+            'unit3': {
+                'random_flag': True,
+                'private-address': '10.0.0.16'}}
+        relation_get.side_effect = lambda v, rid, unit: rdata[unit].get(v)
+        rget_calls = [
+            mock.call('random_flag', rid='relid1', unit='unit1'),
+            mock.call('private-address', rid='relid1', unit='unit1'),
+            mock.call('random_flag', rid='relid1', unit='unit2'),
+            mock.call('private-address', rid='relid1', unit='unit2'),
+            mock.call('random_flag', rid='relid1', unit='unit3'),
+            mock.call('private-address', rid='relid1', unit='unit3')]
+        self.assertSequenceEqual(
+            utils.get_node_flags('random_flag'),
+            ['10.0.0.10', '10.0.0.16', '10.0.0.34', '10.0.0.41'])
+        relation_get.assert_has_calls(rget_calls)
+
+    @mock.patch.object(utils, 'get_node_flags')
+    def test_get_cluster_nodes(self, get_node_flags):
+        utils.get_cluster_nodes()
+        get_node_flags.assert_called_once_with('ready')
+
+    @mock.patch.object(utils, 'get_node_flags')
+    def test_get_member_ready_nodes(self, get_node_flags):
+        utils.get_member_ready_nodes()
+        get_node_flags.assert_called_once_with('member_ready')