openstack/magnum
Code Issues Proposed changes

Changes Swarm Bootstrapping from Public to Etcd

Browse Source 
Previously, Swarm leveraged Docker's public discovery mechanism
for bootstrapping a cluster. Etcd bootstrapping is supported by
Swarm and is preferred for production use for the following reasons:

1. Required for HA.
2. Is more secure.
3. Required for the Flannel network-driver.

Partially-Implements: blueprint extend-baymodel-net-attributes
Partially-Implements: blueprint conductor-template-net-update

Change-Id: Iab844c03ed7cf8bbee69b72ff71c219f0a5ab1dd
This commit is contained in:
Daneyon Hansen 2015-11-17 22:11:19 +00:00
parent f108b46e79
commit de1edaec40
10 changed files with 105 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
etc/magnum/magnum.conf.sample
View File

@ -322,18 +322,6 @@
# value)
#swarm_atomic_template_path = $pybasedir/templates/swarm/swarm.yaml
# Format string to use for swarm discovery url. Available values:
# bay_id, bay_uuid. Example: "etcd://etcd.example.com/\%(bay_uuid)s"
# (string value)
#swarm_discovery_url_format = <None>
# Indicates Swarm discovery should use public endpoint. (boolean
# value)
#public_swarm_discovery = true
# Url for swarm public discovery endpoint. (string value)
#public_swarm_discovery_url = https://discovery.hub.docker.com/v1/clusters
# Location of template to build a Mesos cluster on Ubuntu. (string
# value)
#mesos_ubuntu_template_path = $pybasedir/templates/mesos/mesoscluster.yaml

35
magnum/conductor/template_definition.py
View File

@ -58,17 +58,6 @@ template_def_opts = [
'swarm.yaml'),
help=_('Location of template to build a swarm '
'cluster on atomic.')),
cfg.StrOpt('swarm_discovery_url_format',
help=_('Format string to use for swarm discovery url. '
'Available values: bay_id, bay_uuid. '
'Example: "etcd://etcd.example.com/\%(bay_uuid)s"')),
cfg.BoolOpt('public_swarm_discovery',
default=True,
help=_('Indicates Swarm discovery should use public '
'endpoint.')),
cfg.StrOpt('public_swarm_discovery_url',
default='https://discovery.hub.docker.com/v1/clusters',
help=_('Url for swarm public discovery endpoint.')),
cfg.StrOpt('mesos_ubuntu_template_path',
default=paths.basedir_def('templates/mesos/'
'mesoscluster.yaml'),
@ -593,24 +582,20 @@ class AtomicSwarmTemplateDefinition(BaseTemplateDefinition):
self.add_output('discovery_url',
bay_attr='discovery_url')
@staticmethod
def get_public_token():
token_id = requests.post(cfg.CONF.bay.public_swarm_discovery_url).text
return 'token://%s' % token_id
@staticmethod
def parse_discovery_url(bay):
strings = dict(bay_id=bay.id, bay_uuid=bay.uuid)
return cfg.CONF.bay.swarm_discovery_url_format % strings
def get_discovery_url(self, bay):
if hasattr(bay, 'discovery_url') and bay.discovery_url:
discovery_url = bay.discovery_url
elif cfg.CONF.bay.public_swarm_discovery:
discovery_url = self.get_public_token()
else:
discovery_url = self.parse_discovery_url(bay)
discovery_endpoint = (
cfg.CONF.bay.etcd_discovery_service_endpoint_format %
{'size': 1})
discovery_url = requests.get(discovery_endpoint).text
if not discovery_url:
raise exception.InvalidDiscoveryURL(
discovery_url=discovery_url,
discovery_endpoint=discovery_endpoint)
else:
bay.discovery_url = discovery_url
return discovery_url
def get_params(self, context, baymodel, bay, **kwargs):

17
magnum/templates/swarm/fragments/configure-etcd.sh Normal file
View File

@ -0,0 +1,17 @@
#!/bin/sh
. /etc/sysconfig/heat-params
myip=$(ip addr show eth0 |
awk '$1 == "inet" {print $2}' | cut -f1 -d/)
cat > /etc/etcd/etcd.conf <<EOF
ETCD_NAME="$myip"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
ETCD_LISTEN_PEER_URLS="http://$myip:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://$myip:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://$myip:2380"
ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
EOF

1
magnum/templates/swarm/fragments/write-heat-params.yaml
View File

@ -6,6 +6,7 @@ write_files:
permissions: "0644"
content: |
WAIT_HANDLE="$WAIT_HANDLE"
ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL"
DOCKER_VOLUME="$DOCKER_VOLUME"
HTTP_PROXY="$HTTP_PROXY"
HTTPS_PROXY="$HTTPS_PROXY"

2
magnum/templates/swarm/fragments/write-swarm-agent-service.yaml
View File

@ -16,7 +16,7 @@ write_files:
ExecStartPre=-/usr/bin/docker kill swarm-agent
ExecStartPre=-/usr/bin/docker rm swarm-agent
ExecStartPre=-/usr/bin/docker pull swarm:1.0.0
ExecStart=/usr/bin/docker run -e http_proxy=$HTTP_PROXY -e https_proxy=$HTTPS_PROXY -e no_proxy=$NO_PROXY --name swarm-agent swarm:1.0.0 join --addr $NODE_IP:2375 $DISCOVERY_URL
ExecStart=/usr/bin/docker run -e http_proxy=$HTTP_PROXY -e https_proxy=$HTTPS_PROXY -e no_proxy=$NO_PROXY --name swarm-agent swarm:1.0.0 join --addr $NODE_IP:2375 etcd://$SWARM_MASTER_IP:2379/v2/keys/swarm/
ExecStop=/usr/bin/docker stop swarm-agent
ExecStartPost=/usr/bin/curl -sf -X PUT -H 'Content-Type: application/json' \
--data-binary '{"Status": "SUCCESS", "Reason": "Setup complete", "Data": "OK", "UniqueId": "00000"}' \

2
magnum/templates/swarm/fragments/write-swarm-master-service.sh
View File

@ -34,7 +34,7 @@ END_TLS
fi
cat >> /etc/systemd/system/swarm-manager.service << END_SERVICE_BOTTOM
$DISCOVERY_URL
etcd://$SWARM_MASTER_IP:2379/v2/keys/swarm/
ExecStop=/usr/bin/docker stop swarm-manager
ExecStartPost=/usr/bin/curl -sf -X PUT -H 'Content-Type: application/json' \\
--data-binary '{"Status": "SUCCESS", "Reason": "Setup complete", "Data": "OK", "UniqueId": "00000"}' \\

14
magnum/templates/swarm/swarm.yaml
View File

@ -223,6 +223,7 @@ resources:
template: {get_file: fragments/write-heat-params.yaml}
params:
"$WAIT_HANDLE": {get_resource: cloud_init_wait_handle}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
"$DOCKER_VOLUME": {get_resource: docker_volume}
"$HTTP_PROXY": {get_param: http_proxy}
"$HTTPS_PROXY": {get_param: https_proxy}
@ -256,6 +257,12 @@ resources:
group: ungrouped
config: {get_file: fragments/network-service.sh}
configure_etcd:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-etcd.sh}
configure_swarm:
type: "OS::Heat::SoftwareConfig"
properties:
@ -316,6 +323,7 @@ resources:
str_replace:
template: {get_file: fragments/write-swarm-agent-service.yaml}
params:
"$SWARM_MASTER_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
"$NODE_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
"$DISCOVERY_URL": {get_param: discovery_url}
"$WAIT_HANDLE": {get_resource: agent_wait_handle}
@ -331,6 +339,7 @@ resources:
str_replace:
template: {get_file: fragments/write-swarm-master-service.sh}
params:
"$SWARM_MASTER_IP": {get_attr: [swarm_master_eth0, fixed_ips, 0, ip_address]}
"$DISCOVERY_URL": {get_param: discovery_url}
"$WAIT_HANDLE": {get_resource: master_wait_handle}
"$HTTP_PROXY": {get_param: http_proxy}
@ -346,7 +355,7 @@ resources:
str_replace:
template: {get_file: fragments/enable-services.sh}
params:
"$NODE_SERVICES": "docker.socket swarm-agent swarm-manager"
"$NODE_SERVICES": "etcd docker.socket swarm-agent swarm-manager"
cfn_signal:
type: "OS::Heat::SoftwareConfig"
@ -380,10 +389,11 @@ resources:
- config: {get_resource: remove_docker_key}
- config: {get_resource: write_heat_params}
- config: {get_resource: make_cert}
- config: {get_resource: configure_etcd}
- config: {get_resource: configure_docker_storage}
- config: {get_resource: write_network_config}
- config: {get_resource: network_config_service}
- config: {get_resource: network_service}
- config: {get_resource: configure_docker_storage}
- config: {get_resource: write_swarm_agent_failure_service}
- config: {get_resource: write_swarm_manager_failure_service}
- config: {get_resource: write_docker_service}

1
magnum/templates/swarm/swarmnode.yaml
View File

@ -212,6 +212,7 @@ resources:
template: {get_file: fragments/write-swarm-agent-service.yaml}
params:
"$NODE_IP": {get_attr: [swarm_node_eth0, fixed_ips, 0, ip_address]}
"$SWARM_MASTER_IP": {get_param: swarm_master_ip}
"$DISCOVERY_URL": {get_param: discovery_url}
"$WAIT_HANDLE": {get_resource: node_agent_wait_handle}
"$HTTP_PROXY": {get_param: http_proxy}

12
magnum/tests/unit/conductor/handlers/test_swarm_bay_conductor.py
View File

@ -20,7 +20,6 @@ from magnum.tests import base
import mock
from mock import patch
from oslo_config import cfg
class TestBayConductorWithSwarm(base.TestCase):
@ -51,7 +50,7 @@ class TestBayConductorWithSwarm(base.TestCase):
'api_address': '172.17.2.3',
'node_addresses': ['172.17.2.4'],
'node_count': 1,
'discovery_url': 'token://39987da72f8386e0d0225ae8929e7cb4',
'discovery_url': 'https://discovery.test.io/123456789',
}
osc_patcher = mock.patch('magnum.common.clients.OpenStackClients')
self.mock_osc_class = osc_patcher.start()
@ -83,7 +82,7 @@ class TestBayConductorWithSwarm(base.TestCase):
'number_of_nodes': '1',
'docker_volume_size': 20,
'fixed_network_cidr': '10.2.0.0/22',
'discovery_url': 'token://39987da72f8386e0d0225ae8929e7cb4',
'discovery_url': 'https://discovery.test.io/123456789',
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'no_proxy': 'no_proxy',
@ -99,16 +98,13 @@ class TestBayConductorWithSwarm(base.TestCase):
def test_extract_template_definition_only_required(
self,
mock_objects_baymodel_get_by_uuid):
cfg.CONF.set_override('public_swarm_discovery', False, group='bay')
cfg.CONF.set_override('swarm_discovery_url_format',
'test_discovery', group='bay')
not_required = ['image_id', 'flavor_id', 'dns_nameserver',
'docker_volume_size', 'fixed_network', 'http_proxy',
'https_proxy', 'no_proxy']
for key in not_required:
self.baymodel_dict[key] = None
self.bay_dict['discovery_url'] = None
self.bay_dict['discovery_url'] = 'https://discovery.etcd.io/test'
baymodel = objects.BayModel(self.context, **self.baymodel_dict)
mock_objects_baymodel_get_by_uuid.return_value = baymodel
@ -122,7 +118,7 @@ class TestBayConductorWithSwarm(base.TestCase):
'ssh_key_name': 'keypair_id',
'external_network': 'external_network_id',
'number_of_nodes': '1',
'discovery_url': 'test_discovery',
'discovery_url': 'https://discovery.etcd.io/test',
'user_token': 'fake_token',
'bay_uuid': 'some_uuid',
'magnum_url': self.mock_osc.magnum_url.return_value,

105
magnum/tests/unit/conductor/test_template_definition.py
View File

@ -371,63 +371,74 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
@mock.patch('requests.post')
def test_swarm_discovery_url_public_token(self, mock_post):
@mock.patch('magnum.common.clients.OpenStackClients')
@mock.patch('magnum.conductor.template_definition'
'.AtomicSwarmTemplateDefinition.get_discovery_url')
@mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition'
'.get_params')
@mock.patch('magnum.conductor.template_definition.TemplateDefinition'
'.get_output')
def test_swarm_get_params(self, mock_get_output, mock_get_params,
mock_get_discovery_url, mock_osc_class):
mock_context = mock.MagicMock()
mock_context.auth_token = 'AUTH_TOKEN'
mock_baymodel = mock.MagicMock()
mock_baymodel.tls_disabled = False
mock_bay = mock.MagicMock()
mock_bay.uuid = 'bay-xx-xx-xx-xx'
del mock_bay.stack_id
mock_osc = mock.MagicMock()
mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
mock_osc_class.return_value = mock_osc
mock_get_discovery_url.return_value = 'fake_discovery_url'
mock_context.auth_url = 'http://192.168.10.10:5000/v3'
mock_context.user_name = 'fake_user'
mock_context.tenant = 'fake_tenant'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
swarm_def.get_params(mock_context, mock_baymodel, mock_bay)
expected_kwargs = {'extra_params': {
'discovery_url': 'fake_discovery_url',
'user_token': mock_context.auth_token,
'magnum_url': mock_osc.magnum_url.return_value}}
mock_get_params.assert_called_once_with(mock_context, mock_baymodel,
mock_bay, **expected_kwargs)
@mock.patch('requests.get')
def test_swarm_get_discovery_url(self, mock_get):
cfg.CONF.set_override('etcd_discovery_service_endpoint_format',
'http://etcd/test?size=%(size)d',
group='bay')
expected_discovery_url = 'http://etcd/token'
mock_resp = mock.MagicMock()
mock_resp.text = 'some_token'
mock_post.return_value = mock_resp
mock_resp.text = expected_discovery_url
mock_get.return_value = mock_resp
mock_bay = mock.MagicMock()
mock_bay.discovery_url = None
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
discovery_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual('token://some_token', actual_url)
mock_get.assert_called_once_with('http://etcd/test?size=1')
self.assertEqual(mock_bay.discovery_url, expected_discovery_url)
self.assertEqual(discovery_url, expected_discovery_url)
def test_swarm_discovery_url_format_bay_id(self):
cfg.CONF.set_override('public_swarm_discovery', False, group='bay')
cfg.CONF.set_override('swarm_discovery_url_format',
'etcd://test.com/bay-%(bay_id)s', group='bay')
@mock.patch('requests.get')
def test_swarm_get_discovery_url_not_found(self, mock_get):
mock_resp = mock.MagicMock()
mock_resp.text = ''
mock_get.return_value = mock_resp
mock_bay = mock.MagicMock()
mock_bay.discovery_url = None
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
fake_bay = mock.MagicMock()
fake_bay.discovery_url = None
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual('etcd://test.com/bay-1', actual_url)
def test_swarm_discovery_url_format_bay_uuid(self):
cfg.CONF.set_override('public_swarm_discovery', False, group='bay')
cfg.CONF.set_override('swarm_discovery_url_format',
'etcd://test.com/bay-%(bay_uuid)s', group='bay')
mock_bay = mock.MagicMock()
mock_bay.discovery_url = None
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual('etcd://test.com/bay-some_uuid', actual_url)
def test_swarm_discovery_url_from_bay(self):
mock_bay = mock.MagicMock()
mock_bay.discovery_url = 'token://some_token'
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual(mock_bay.discovery_url, actual_url)
self.assertRaises(exception.InvalidDiscoveryURL,
tdef.AtomicK8sTemplateDefinition().get_discovery_url,
fake_bay)
def test_swarm_get_heat_param(self):
swarm_def = tdef.AtomicSwarmTemplateDefinition()