From bcdd70cf1e1fd1c1c52587202bf5a9a38703a44d Mon Sep 17 00:00:00 2001 From: Hongbin Lu Date: Tue, 21 Jul 2015 23:37:53 -0400 Subject: [PATCH] Bootstrap etcd cluster by discovery_url * Configure etcd to use a discovery_url to bootstrap the cluster. * Users can provide discovery_url for individual bay. * If discovery_url is not provided, it will be generated at runtime by using a discovery service. * Admin can set the endpoint of the discovery service in config file. Default is the public etcd discovery service. Change-Id: I9dd3a47f6d50ebadf74c4ee65701183f18c9d629 Partially-Implements: blueprint make-master-ha --- etc/magnum/magnum.conf.sample | 3 ++ magnum/conductor/template_definition.py | 19 +++++++ .../fragments/configure-etcd.sh | 10 +++- .../fragments/write-heat-params-master.yaml | 1 + .../heat-kubernetes/kubecluster.yaml | 6 +++ .../conductor/handlers/test_bay_conductor.py | 49 +++++++++++++++++-- .../conductor/test_template_definition.py | 9 +++- 7 files changed, 90 insertions(+), 7 deletions(-) diff --git a/etc/magnum/magnum.conf.sample b/etc/magnum/magnum.conf.sample index 1a285eaeda..ec6f85baf1 100644 --- a/etc/magnum/magnum.conf.sample +++ b/etc/magnum/magnum.conf.sample @@ -252,6 +252,9 @@ # value) #k8s_coreos_template_path = $pybasedir/templates/heat-kubernetes/kubecluster-coreos.yaml +# Url for etcd public discovery endpoint. (string value) +#etcd_discovery_service_endpoint_format = https://discovery.etcd.io/new?size=%(size)d + # coreos discovery token url. (string value) # Deprecated group/name - [bay_heat]/discovery_token_url #coreos_discovery_token_url = diff --git a/magnum/conductor/template_definition.py b/magnum/conductor/template_definition.py index 7d11e0a72e..97a8f33cbb 100644 --- a/magnum/conductor/template_definition.py +++ b/magnum/conductor/template_definition.py @@ -42,6 +42,9 @@ template_def_opts = [ 'kubecluster-coreos.yaml'), help=_( 'Location of template to build a k8s cluster on CoreOS.')), + cfg.StrOpt('etcd_discovery_service_endpoint_format', + default='https://discovery.etcd.io/new?size=%(size)d', + help=_('Url for etcd public discovery endpoint.')), cfg.StrOpt('coreos_discovery_token_url', default=None, deprecated_name='discovery_token_url', @@ -360,6 +363,20 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition): self.add_output('kube_minions_external', bay_attr='node_addresses') + def get_discovery_url(self, bay): + if hasattr(bay, 'discovery_url') and bay.discovery_url: + discovery_url = bay.discovery_url + else: + # TODO(hongbin): Eliminate hard coding of the size when multiple + # masters is supported. + discovery_endpoint = ( + cfg.CONF.bay.etcd_discovery_service_endpoint_format % + {'size': 1}) + discovery_url = requests.get(discovery_endpoint).text + bay.discovery_url = discovery_url + + return discovery_url + def get_params(self, context, baymodel, bay, **kwargs): extra_params = kwargs.pop('extra_params', {}) scale_mgr = kwargs.pop('scale_manager', None) @@ -368,6 +385,8 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition): extra_params['minions_to_remove'] = ( scale_mgr.get_removal_nodes(hosts)) + extra_params['discovery_url'] = self.get_discovery_url(bay) + return super(AtomicK8sTemplateDefinition, self).get_params(context, baymodel, bay, extra_params=extra_params, diff --git a/magnum/templates/heat-kubernetes/fragments/configure-etcd.sh b/magnum/templates/heat-kubernetes/fragments/configure-etcd.sh index 00335c2449..980a345c20 100644 --- a/magnum/templates/heat-kubernetes/fragments/configure-etcd.sh +++ b/magnum/templates/heat-kubernetes/fragments/configure-etcd.sh @@ -1,13 +1,19 @@ #!/bin/sh +. /etc/sysconfig/heat-params + myip=$(ip addr show eth0 | awk '$1 == "inet" {print $2}' | cut -f1 -d/) cat > /etc/etcd/etcd.conf < + Discovery URL used for bootstrapping the etcd cluster. + resources: master_wait_handle: @@ -191,6 +196,7 @@ resources: "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} "$FLANNEL_USE_VXLAN": {get_param: flannel_use_vxlan} "$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr} + "$ETCD_DISCOVERY_URL": {get_param: discovery_url} configure_etcd: type: OS::Heat::SoftwareConfig diff --git a/magnum/tests/unit/conductor/handlers/test_bay_conductor.py b/magnum/tests/unit/conductor/handlers/test_bay_conductor.py index c79afe26d4..46896025b9 100644 --- a/magnum/tests/unit/conductor/handlers/test_bay_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_bay_conductor.py @@ -52,6 +52,7 @@ class TestBayConductorWithK8s(base.TestCase): 'api_address': '172.17.2.3', 'node_addresses': ['172.17.2.4'], 'node_count': 1, + 'discovery_url': 'https://discovery.etcd.io/test', } @patch('magnum.objects.BayModel.get_by_uuid') @@ -85,7 +86,8 @@ class TestBayConductorWithK8s(base.TestCase): 'fixed_network': 'fixed_network_cidr', 'master_flavor_id': 'master_flavor', 'apiserver_port': '', - 'node_count': 'number_of_minions' + 'node_count': 'number_of_minions', + 'discovery_url': 'discovery_url', } expected = { 'ssh_key_name': 'keypair_id', @@ -97,6 +99,7 @@ class TestBayConductorWithK8s(base.TestCase): 'number_of_minions': '1', 'fixed_network_cidr': '10.20.30.0/24', 'docker_volume_size': 20, + 'discovery_url': 'https://discovery.etcd.io/test', } if missing_attr is not None: expected.pop(mapping[missing_attr], None) @@ -135,7 +138,8 @@ class TestBayConductorWithK8s(base.TestCase): 'fixed_network_cidr': '10.20.30.0/24', 'docker_volume_size': 20, 'ssh_authorized_key': 'ssh_authorized_key', - 'token': 'h3' + 'token': 'h3', + 'discovery_url': 'https://discovery.etcd.io/test', } self.assertEqual(expected, definition) @@ -171,7 +175,8 @@ class TestBayConductorWithK8s(base.TestCase): 'fixed_network_cidr': '10.20.30.0/24', 'docker_volume_size': 20, 'ssh_authorized_key': 'ssh_authorized_key', - 'token': 'ba3d1866282848ddbedc76112110c208' + 'token': 'ba3d1866282848ddbedc76112110c208', + 'discovery_url': 'https://discovery.etcd.io/test', } self.assertEqual(expected, definition) @@ -248,6 +253,7 @@ class TestBayConductorWithK8s(base.TestCase): 'number_of_minions': '1', 'fixed_network_cidr': '10.20.30.0/24', 'docker_volume_size': 20, + 'discovery_url': 'https://discovery.etcd.io/test', } self.assertIn('token', definition) del definition['token'] @@ -269,6 +275,43 @@ class TestBayConductorWithK8s(base.TestCase): mock_objects_baymodel_get_by_uuid, missing_attr='node_count') + @patch('requests.get') + @patch('magnum.objects.BayModel.get_by_uuid') + def test_extract_template_definition_without_discovery_url( + self, + mock_objects_baymodel_get_by_uuid, + reqget): + baymodel = objects.BayModel(self.context, **self.baymodel_dict) + mock_objects_baymodel_get_by_uuid.return_value = baymodel + bay_dict = self.bay_dict + bay_dict['discovery_url'] = None + bay = objects.Bay(self.context, **bay_dict) + + cfg.CONF.set_override('etcd_discovery_service_endpoint_format', + 'http://etcd/test?size=%(size)d', + group='bay') + mock_req = mock.MagicMock(text='https://address/token') + reqget.return_value = mock_req + + (template_path, + definition) = bay_conductor._extract_template_definition(self.context, + bay) + + expected = { + 'ssh_key_name': 'keypair_id', + 'external_network': 'external_network_id', + 'dns_nameserver': 'dns_nameserver', + 'server_image': 'image_id', + 'master_flavor': 'master_flavor_id', + 'minion_flavor': 'flavor_id', + 'number_of_minions': '1', + 'fixed_network_cidr': '10.20.30.0/24', + 'docker_volume_size': 20, + 'discovery_url': 'https://address/token', + } + self.assertEqual(expected, definition) + reqget.assert_called_once_with('http://etcd/test?size=1') + @patch('magnum.objects.BayModel.get_by_uuid') def test_update_stack_outputs(self, mock_objects_baymodel_get_by_uuid): baymodel_dict = self.baymodel_dict diff --git a/magnum/tests/unit/conductor/test_template_definition.py b/magnum/tests/unit/conductor/test_template_definition.py index 878f7f4fe8..57ff749cf1 100644 --- a/magnum/tests/unit/conductor/test_template_definition.py +++ b/magnum/tests/unit/conductor/test_template_definition.py @@ -139,11 +139,14 @@ class TemplateDefinitionTestCase(base.TestCase): class AtomicK8sTemplateDefinitionTestCase(base.TestCase): + @mock.patch('magnum.conductor.template_definition' + '.AtomicK8sTemplateDefinition.get_discovery_url') @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition' '.get_params') @mock.patch('magnum.conductor.template_definition.TemplateDefinition' '.get_output') - def test_k8s_get_params(self, mock_get_output, mock_get_params): + def test_k8s_get_params(self, mock_get_output, mock_get_params, + mock_get_discovery_url): mock_context = mock.MagicMock() mock_baymodel = mock.MagicMock() mock_bay = mock.MagicMock() @@ -151,13 +154,15 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase): removal_nodes = ['node1', 'node2'] mock_scale_manager.get_removal_nodes.return_value = removal_nodes + mock_get_discovery_url.return_value = 'fake_discovery_url' k8s_def = tdef.AtomicK8sTemplateDefinition() k8s_def.get_params(mock_context, mock_baymodel, mock_bay, scale_manager=mock_scale_manager) expected_kwargs = {'extra_params': { - 'minions_to_remove': removal_nodes}} + 'minions_to_remove': removal_nodes, + 'discovery_url': 'fake_discovery_url'}} mock_get_params.assert_called_once_with(mock_context, mock_baymodel, mock_bay, **expected_kwargs)