Bootstrap etcd cluster by discovery_url

* Configure etcd to use a discovery_url to bootstrap the cluster.
* Users can provide discovery_url for individual bay.
* If discovery_url is not provided, it will be generated at runtime
  by using a discovery service.
* Admin can set the endpoint of the discovery service in config file.
  Default is the public etcd discovery service.

Change-Id: I9dd3a47f6d50ebadf74c4ee65701183f18c9d629
Partially-Implements: blueprint make-master-ha

etc/magnum/magnum.conf.sample

@@ -252,6 +252,9 @@
 # value)
 #k8s_coreos_template_path = $pybasedir/templates/heat-kubernetes/kubecluster-coreos.yaml
 
+# Url for etcd public discovery endpoint. (string value)
+#etcd_discovery_service_endpoint_format = https://discovery.etcd.io/new?size=%(size)d
+
 # coreos discovery token url. (string value)
 # Deprecated group/name - [bay_heat]/discovery_token_url
 #coreos_discovery_token_url = <None>

magnum/conductor/template_definition.py

@@ -42,6 +42,9 @@ template_def_opts = [
                                          'kubecluster-coreos.yaml'),
                help=_(
                    'Location of template to build a k8s cluster on CoreOS.')),
+    cfg.StrOpt('etcd_discovery_service_endpoint_format',
+               default='https://discovery.etcd.io/new?size=%(size)d',
+               help=_('Url for etcd public discovery endpoint.')),
     cfg.StrOpt('coreos_discovery_token_url',
                default=None,
                deprecated_name='discovery_token_url',
@@ -360,6 +363,20 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
         self.add_output('kube_minions_external',
                         bay_attr='node_addresses')
 
+    def get_discovery_url(self, bay):
+        if hasattr(bay, 'discovery_url') and bay.discovery_url:
+            discovery_url = bay.discovery_url
+        else:
+            # TODO(hongbin): Eliminate hard coding of the size when multiple
+            # masters is supported.
+            discovery_endpoint = (
+                cfg.CONF.bay.etcd_discovery_service_endpoint_format %
+                {'size': 1})
+            discovery_url = requests.get(discovery_endpoint).text
+            bay.discovery_url = discovery_url
+
+        return discovery_url
+
     def get_params(self, context, baymodel, bay, **kwargs):
         extra_params = kwargs.pop('extra_params', {})
         scale_mgr = kwargs.pop('scale_manager', None)
@@ -368,6 +385,8 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
             extra_params['minions_to_remove'] = (
                 scale_mgr.get_removal_nodes(hosts))
 
+        extra_params['discovery_url'] = self.get_discovery_url(bay)
+
         return super(AtomicK8sTemplateDefinition,
                      self).get_params(context, baymodel, bay,
                                       extra_params=extra_params,

magnum/templates/heat-kubernetes/fragments/configure-etcd.sh

@@ -1,13 +1,19 @@
 #!/bin/sh
 
+. /etc/sysconfig/heat-params
+
 myip=$(ip addr show eth0 |
 awk '$1 == "inet" {print $2}' | cut -f1 -d/)
 
 cat > /etc/etcd/etcd.conf <<EOF
 # [member]
-ETCD_NAME=default
+ETCD_NAME="$myip"
 ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
 ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:4001"
+ETCD_LISTEN_PEER_URLS="http://$myip:7001"
+
 [cluster]
 ETCD_ADVERTISE_CLIENT_URLS="http://$myip:4001"
-EOF
+ETCD_INITIAL_ADVERTISE_PEER_URLS="http://$myip:7001"
+ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
+EOF

magnum/templates/heat-kubernetes/fragments/write-heat-params-master.yaml

@@ -10,3 +10,4 @@ write_files:
       FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN"
       FLANNEL_USE_VXLAN="$FLANNEL_USE_VXLAN"
       PORTAL_NETWORK_CIDR="$PORTAL_NETWORK_CIDR"
+      ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL"

magnum/templates/heat-kubernetes/kubecluster.yaml

@@ -100,6 +100,11 @@ parameters:
       be empty when doing an create.
     default: []
 
+  discovery_url:
+    type: string
+    description: >
+      Discovery URL used for bootstrapping the etcd cluster.
+
 resources:
 
   master_wait_handle:
@@ -191,6 +196,7 @@ resources:
             "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
             "$FLANNEL_USE_VXLAN": {get_param: flannel_use_vxlan}
             "$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
+            "$ETCD_DISCOVERY_URL": {get_param: discovery_url}
 
   configure_etcd:
     type: OS::Heat::SoftwareConfig

magnum/tests/unit/conductor/handlers/test_bay_conductor.py

@@ -52,6 +52,7 @@ class TestBayConductorWithK8s(base.TestCase):
             'api_address': '172.17.2.3',
             'node_addresses': ['172.17.2.4'],
             'node_count': 1,
+            'discovery_url': 'https://discovery.etcd.io/test',
         }
 
     @patch('magnum.objects.BayModel.get_by_uuid')
@@ -85,7 +86,8 @@ class TestBayConductorWithK8s(base.TestCase):
             'fixed_network': 'fixed_network_cidr',
             'master_flavor_id': 'master_flavor',
             'apiserver_port': '',
-            'node_count': 'number_of_minions'
+            'node_count': 'number_of_minions',
+            'discovery_url': 'discovery_url',
         }
         expected = {
             'ssh_key_name': 'keypair_id',
@@ -97,6 +99,7 @@ class TestBayConductorWithK8s(base.TestCase):
             'number_of_minions': '1',
             'fixed_network_cidr': '10.20.30.0/24',
             'docker_volume_size': 20,
+            'discovery_url': 'https://discovery.etcd.io/test',
         }
         if missing_attr is not None:
             expected.pop(mapping[missing_attr], None)
@@ -135,7 +138,8 @@ class TestBayConductorWithK8s(base.TestCase):
             'fixed_network_cidr': '10.20.30.0/24',
             'docker_volume_size': 20,
             'ssh_authorized_key': 'ssh_authorized_key',
-            'token': 'h3'
+            'token': 'h3',
+            'discovery_url': 'https://discovery.etcd.io/test',
         }
         self.assertEqual(expected, definition)
 
@@ -171,7 +175,8 @@ class TestBayConductorWithK8s(base.TestCase):
             'fixed_network_cidr': '10.20.30.0/24',
             'docker_volume_size': 20,
             'ssh_authorized_key': 'ssh_authorized_key',
-            'token': 'ba3d1866282848ddbedc76112110c208'
+            'token': 'ba3d1866282848ddbedc76112110c208',
+            'discovery_url': 'https://discovery.etcd.io/test',
         }
         self.assertEqual(expected, definition)
 
@@ -248,6 +253,7 @@ class TestBayConductorWithK8s(base.TestCase):
             'number_of_minions': '1',
             'fixed_network_cidr': '10.20.30.0/24',
             'docker_volume_size': 20,
+            'discovery_url': 'https://discovery.etcd.io/test',
         }
         self.assertIn('token', definition)
         del definition['token']
@@ -269,6 +275,43 @@ class TestBayConductorWithK8s(base.TestCase):
             mock_objects_baymodel_get_by_uuid,
             missing_attr='node_count')
 
+    @patch('requests.get')
+    @patch('magnum.objects.BayModel.get_by_uuid')
+    def test_extract_template_definition_without_discovery_url(
+            self,
+            mock_objects_baymodel_get_by_uuid,
+            reqget):
+        baymodel = objects.BayModel(self.context, **self.baymodel_dict)
+        mock_objects_baymodel_get_by_uuid.return_value = baymodel
+        bay_dict = self.bay_dict
+        bay_dict['discovery_url'] = None
+        bay = objects.Bay(self.context, **bay_dict)
+
+        cfg.CONF.set_override('etcd_discovery_service_endpoint_format',
+                              'http://etcd/test?size=%(size)d',
+                              group='bay')
+        mock_req = mock.MagicMock(text='https://address/token')
+        reqget.return_value = mock_req
+
+        (template_path,
+         definition) = bay_conductor._extract_template_definition(self.context,
+                                                                  bay)
+
+        expected = {
+            'ssh_key_name': 'keypair_id',
+            'external_network': 'external_network_id',
+            'dns_nameserver': 'dns_nameserver',
+            'server_image': 'image_id',
+            'master_flavor': 'master_flavor_id',
+            'minion_flavor': 'flavor_id',
+            'number_of_minions': '1',
+            'fixed_network_cidr': '10.20.30.0/24',
+            'docker_volume_size': 20,
+            'discovery_url': 'https://address/token',
+        }
+        self.assertEqual(expected, definition)
+        reqget.assert_called_once_with('http://etcd/test?size=1')
+
     @patch('magnum.objects.BayModel.get_by_uuid')
     def test_update_stack_outputs(self, mock_objects_baymodel_get_by_uuid):
         baymodel_dict = self.baymodel_dict

magnum/tests/unit/conductor/test_template_definition.py

@@ -139,11 +139,14 @@ class TemplateDefinitionTestCase(base.TestCase):
 
 class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
 
+    @mock.patch('magnum.conductor.template_definition'
+                '.AtomicK8sTemplateDefinition.get_discovery_url')
     @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition'
                 '.get_params')
     @mock.patch('magnum.conductor.template_definition.TemplateDefinition'
                 '.get_output')
-    def test_k8s_get_params(self, mock_get_output, mock_get_params):
+    def test_k8s_get_params(self, mock_get_output, mock_get_params,
+                            mock_get_discovery_url):
         mock_context = mock.MagicMock()
         mock_baymodel = mock.MagicMock()
         mock_bay = mock.MagicMock()
@@ -151,13 +154,15 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
 
         removal_nodes = ['node1', 'node2']
         mock_scale_manager.get_removal_nodes.return_value = removal_nodes
+        mock_get_discovery_url.return_value = 'fake_discovery_url'
         k8s_def = tdef.AtomicK8sTemplateDefinition()
 
         k8s_def.get_params(mock_context, mock_baymodel, mock_bay,
                            scale_manager=mock_scale_manager)
 
         expected_kwargs = {'extra_params': {
-            'minions_to_remove': removal_nodes}}
+            'minions_to_remove': removal_nodes,
+            'discovery_url': 'fake_discovery_url'}}
         mock_get_params.assert_called_once_with(mock_context, mock_baymodel,
                                                 mock_bay, **expected_kwargs)