Add Swarm TemplateDefinition

This change will allow deployers to select either Kubernetes
or Swarm to be the CoE used in Magnum's bays. A Swarm bay uses
a subset of the BayModel parameters used for Kubernetes.
Node discovery is provided via Docker's public discovery
endpoint, but operators and users can override this with
Bay's discovery_url argument.

Implements: bp multiple-bay-templates

Change-Id: I5278e6d477298085d07673810e5d8813d21b7730
This commit is contained in:
Andrew Melton 2015-04-14 20:42:27 -07:00
parent 784643e142
commit 6d64188d50
17 changed files with 328 additions and 28 deletions

View File

@ -259,6 +259,35 @@ Now log into one of the other container hosts and access a redis slave from ther
There are four redis instances, one master and three slaves, running across the bay,
replicating data between one another.
Building a Swarm bay
====================
First, we will need to reconfigure Magnum. We need to set 'cluster_coe' in
the 'k8s_head' section to 'swarm' in the magnum.conf. After changing
magnum.conf restart magnum-api and magnum-conductor.::
sudo cat >> /etc/magnum/magnum.conf << END_CONFIG
[k8s_heat]
cluster_coe=swarm
END_CONFIG
Next, create a baymodel, it is very similar to the Kubernetes baymodel,
it is only missing some Kubernetes specific arguments.::
NIC_ID=$(neutron net-show public | awk '/ id /{print $4}')
magnum baymodel-create --name swarmbaymodel --image-id fedora-21-atomic-2 \
--keypair-id testkey \
--external-network-id $NIC_ID \
--dns-nameserver 8.8.8.8 --flavor-id m1.small
Finally, create the bay. Use the baymodel 'swarmbaymodel' as a template for
bay creation. This bay will result in one swarm manager node and two extra
agent nodes. ::
magnum bay-create --name swarmbay --baymodel swarmbaymodel --node-count 2
Building developer documentation
================================

View File

@ -229,8 +229,12 @@
# Deprecated group/name - [k8s_heat]/discovery_token_url
#coreos_discovery_token_url = <None>
# Location of template to build a swarm cluster on atomic. (string
# value)
#swarm_atomic_template_path = $pybasedir/templates/docker-swarm/swarm.yaml
# Enabled bay definition entry points. (list value)
#enabled_definitions = magnum_vm_atomic_k8s,magnum_vm_coreos_k8s
#enabled_definitions = magnum_vm_atomic_k8s,magnum_vm_coreos_k8s,magnum_vm_atomic_swarm
[conductor]
@ -430,6 +434,10 @@
# Cluster types are fedora-atomic, coreos, ironic. (string value)
#cluster_type = fedora-atomic
# Container Orchestration Environments are kubernetes or swarm.
# (string value)
#cluster_coe = kubernetes
# Number of attempts to query the Heat stack for finding out the
# status of the created stack and getting url of the DU created in the
# stack (integer value)

View File

@ -81,6 +81,9 @@ class Bay(base.APIBase):
status = wtypes.text
"""Status of the bay from the heat stack"""
discovery_url = wtypes.text
"""Url used for bay node discovery"""
def __init__(self, **kwargs):
super(Bay, self).__init__()

View File

@ -19,7 +19,7 @@ from oslo_config import cfg
from magnum.common import clients
from magnum.common import exception
from magnum.common import short_id
from magnum.conductor import template_definition as tdef
from magnum.conductor.template_definition import TemplateDefinition as TDef
from magnum import objects
from magnum.openstack.common._i18n import _
from magnum.openstack.common._i18n import _LE
@ -32,6 +32,10 @@ k8s_heat_opts = [
cfg.StrOpt('cluster_type',
default='fedora-atomic',
help=_('Cluster types are fedora-atomic, coreos, ironic.')),
cfg.StrOpt('cluster_coe',
default='kubernetes',
help=_('Container Orchestration Environments are '
'kubernetes or swarm. ')),
cfg.IntOpt('max_attempts',
default=2000,
help=('Number of attempts to query the Heat stack for '
@ -51,9 +55,9 @@ LOG = logging.getLogger(__name__)
def _extract_template_definition(context, bay):
baymodel = objects.BayModel.get_by_uuid(context, bay.baymodel_id)
definition = tdef.TemplateDefinition.get_template_definition('vm',
cfg.CONF.k8s_heat.cluster_type,
'kubernetes')
cluster_type = cfg.CONF.k8s_heat.cluster_type
cluster_coe = cfg.CONF.k8s_heat.cluster_coe
definition = TDef.get_template_definition('vm', cluster_type, cluster_coe)
return definition.extract_definition(baymodel, bay)
@ -88,9 +92,9 @@ def _update_stack(context, osc, bay):
def _update_stack_outputs(stack, bay):
definition = tdef.TemplateDefinition.get_template_definition('vm',
cfg.CONF.k8s_heat.cluster_type,
'kubernetes')
cluster_type = cfg.CONF.k8s_heat.cluster_type
cluster_coe = cfg.CONF.k8s_heat.cluster_coe
definition = TDef.get_template_definition('vm', cluster_type, cluster_coe)
return definition.update_outputs(stack, bay)

View File

@ -43,8 +43,26 @@ template_def_opts = [
deprecated_name='discovery_token_url',
deprecated_group='k8s_heat',
help=_('coreos discovery token url.')),
cfg.StrOpt('swarm_atomic_template_path',
default=paths.basedir_def('templates/docker-swarm/'
'swarm.yaml'),
help=_('Location of template to build a swarm '
'cluster on atomic. ')),
cfg.StrOpt('swarm_discovery_url_format',
default=None,
help=_('Format string to use for swarm discovery url. '
'Available values: bay_id, bay_uuid. '
'Example: "etcd://etcd.example.com/\%(bay_uuid)s" ')),
cfg.BoolOpt('public_swarm_discovery',
default=True,
help=_('Indicates Swarm discovery should use public '
'endpoint.')),
cfg.StrOpt('public_swarm_discovery_url',
default='https://discovery-stage.hub.docker.com/v1/clusters',
help=_('Url for swarm public discovery endpoint.')),
cfg.ListOpt('enabled_definitions',
default=['magnum_vm_atomic_k8s', 'magnum_vm_coreos_k8s'],
default=['magnum_vm_atomic_k8s', 'magnum_vm_coreos_k8s',
'magnum_vm_atomic_swarm'],
help=_('Enabled bay definition entry points. ')),
]
@ -263,11 +281,20 @@ class BaseTemplateDefinition(TemplateDefinition):
self.add_parameter('ssh_key_name',
baymodel_attr='keypair_id',
required=True)
self.add_parameter('external_network_id',
baymodel_attr='external_network_id',
required=True)
self.add_parameter('server_image',
baymodel_attr='image_id')
self.add_parameter('server_flavor',
baymodel_attr='flavor_id')
self.add_parameter('dns_nameserver',
baymodel_attr='dns_nameserver')
@abc.abstractproperty
def template_path(self):
pass
class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
@ -277,12 +304,6 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
def __init__(self):
super(AtomicK8sTemplateDefinition, self).__init__()
self.add_parameter('external_network_id',
baymodel_attr='external_network_id',
required=True)
self.add_parameter('dns_nameserver',
baymodel_attr='dns_nameserver')
self.add_parameter('master_flavor',
baymodel_attr='master_flavor_id')
self.add_parameter('fixed_network',
@ -339,3 +360,57 @@ class CoreOSK8sTemplateDefinition(AtomicK8sTemplateDefinition):
@property
def template_path(self):
return cfg.CONF.bay.k8s_coreos_template_path
class AtomicSwarmTemplateDefinition(BaseTemplateDefinition):
provides = [
{'platform': 'vm', 'os': 'fedora-atomic', 'coe': 'swarm'},
]
def __init__(self):
super(AtomicSwarmTemplateDefinition, self).__init__()
self.add_parameter('number_of_nodes',
bay_attr='node_count',
param_type=str)
self.add_parameter('fixed_network_cidr',
baymodel_attr='fixed_network')
self.add_output('swarm_manager',
bay_attr='api_address')
self.add_output('swarm_nodes_external',
bay_attr='node_addresses')
self.add_output('discovery_url',
bay_attr='discovery_url')
@staticmethod
def get_public_token():
token_id = requests.post(cfg.CONF.bay.public_swarm_discovery_url).text
return 'token://%s' % token_id
@staticmethod
def parse_discovery_url(bay):
strings = dict(bay_id=bay.id, bay_uuid=bay.uuid)
return cfg.CONF.bay.swarm_discovery_url_format % strings
def get_discovery_url(self, bay):
if hasattr(bay, 'discovery_url') and bay.discovery_url:
discovery_url = bay.discovery_url
elif cfg.CONF.bay.public_swarm_discovery:
discovery_url = self.get_public_token()
else:
discovery_url = self.parse_discovery_url(bay)
return discovery_url
def get_params(self, baymodel, bay, extra_params=None):
if not extra_params:
extra_params = dict()
extra_params['discovery_url'] = self.get_discovery_url(bay)
return super(AtomicSwarmTemplateDefinition,
self).get_params(baymodel, bay, extra_params=extra_params)
@property
def template_path(self):
return cfg.CONF.bay.swarm_atomic_template_path

View File

@ -0,0 +1,31 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add-discovery-url-to-bay
Revision ID: 4ea34a59a64c
Revises: 456126c6c9e9
Create Date: 2015-04-14 18:56:03.440329
"""
# revision identifiers, used by Alembic.
revision = '4ea34a59a64c'
down_revision = '456126c6c9e9'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('bay',
sa.Column('discovery_url', sa.String(length=255),
nullable=True))

View File

@ -129,6 +129,7 @@ class Bay(Base):
node_addresses = Column(JSONEncodedList)
node_count = Column(Integer())
status = Column(String(20), nullable=True)
discovery_url = Column(String(255))
class BayLock(Base):

View File

@ -52,7 +52,8 @@ class Bay(base.MagnumObject):
'status': obj_utils.str_or_none,
'api_address': obj_utils.str_or_none,
'node_addresses': obj_utils.list_or_none,
'node_count': obj_utils.int_or_none
'node_count': obj_utils.int_or_none,
'discovery_url': obj_utils.str_or_none,
}
@staticmethod

View File

@ -40,7 +40,7 @@ your environment:
external_network_id: 028d70dd-67b8-4901-8bdd-0c62b06cce2d
dns_nameserver: 192.168.200.1
server_image: fedora-21-atomic
swarm_token: d8cdfe5128af6e1075b34aa06ff1cc2c
discovery_url: token://d8cdfe5128af6e1075b34aa06ff1cc2c
And then create the stack, referencing that environment file:
@ -51,7 +51,7 @@ You must provide values for:
- `ssh_key_name`
- `external_network_id`
- `server_image`
- `swarm_token`
- `discovery_url`
## Interacting with Swarm

View File

@ -15,7 +15,7 @@ write_files:
ExecStartPre=-/usr/bin/docker kill swarm-agent
ExecStartPre=-/usr/bin/docker rm swarm-agent
ExecStartPre=/usr/bin/docker pull swarm
ExecStart=/usr/bin/docker run --name swarm-agent swarm join --addr $NODE_IP:2375 token://$SWARM_TOKEN
ExecStart=/usr/bin/docker run --name swarm-agent swarm join --addr $NODE_IP:2375 $DISCOVERY_URL
ExecStop=/usr/bin/docker stop swarm-agent
[Install]

View File

@ -15,7 +15,7 @@ write_files:
ExecStartPre=-/usr/bin/docker kill swarm-manager
ExecStartPre=-/usr/bin/docker rm swarm-manager
ExecStartPre=/usr/bin/docker pull swarm
ExecStart=/usr/bin/docker run --name swarm-manager -p 2376:2375 swarm manage -H tcp://0.0.0.0:2375 token://$SWARM_TOKEN
ExecStart=/usr/bin/docker run --name swarm-manager -p 2376:2375 swarm manage -H tcp://0.0.0.0:2375 $DISCOVERY_URL
ExecStop=/usr/bin/docker stop swarm-manager
[Install]

View File

@ -21,9 +21,9 @@ parameters:
type: string
description: uuid of a network to use for floating ip addresses
swarm_token:
discovery_url:
type: string
description: identifier of swarm cluster to build
description: url provided for node discovery
#
# OPTIONAL PARAMETERS
@ -174,7 +174,7 @@ resources:
template: {get_file: fragments/write-swarm-agent-service.yaml}
params:
"$NODE_IP": {get_attr: [swarm_manager_eth0, fixed_ips, 0, ip_address]}
"$SWARM_TOKEN": {get_param: swarm_token}
"$DISCOVERY_URL": {get_param: discovery_url}
write_swarm_manager_service:
type: "OS::Heat::SoftwareConfig"
@ -184,7 +184,7 @@ resources:
str_replace:
template: {get_file: fragments/write-swarm-manager-service.yaml}
params:
"$SWARM_TOKEN": {get_param: swarm_token}
"$DISCOVERY_URL": {get_param: discovery_url}
enable_services:
type: "OS::Heat::SoftwareConfig"
@ -282,7 +282,7 @@ resources:
fixed_network_id: {get_resource: fixed_network}
fixed_subnet_id: {get_resource: fixed_subnet}
external_network_id: {get_param: external_network_id}
swarm_token: {get_param: swarm_token}
discovery_url: {get_param: discovery_url}
outputs:
@ -294,3 +294,6 @@ outputs:
swarm_nodes_external:
value: {get_attr: [swarm_nodes, swarm_node_external_ip]}
discovery_url:
value: {get_param: discovery_url}

View File

@ -34,9 +34,9 @@ parameters:
type: string
description: Subnet from which to allocate fixed addresses.
swarm_token:
discovery_url:
type: string
description: token of swarm cluster to build
description: url provided for node discovery
resources:
@ -118,7 +118,7 @@ resources:
template: {get_file: fragments/write-swarm-agent-service.yaml}
params:
"$NODE_IP": {get_attr: [swarm_node_eth0, fixed_ips, 0, ip_address]}
"$SWARM_TOKEN": {get_param: swarm_token}
"$DISCOVERY_URL": {get_param: discovery_url}
enable_services:
type: "OS::Heat::SoftwareConfig"

View File

@ -604,3 +604,81 @@ class TestHandler(db_base.DbTestCase):
# The bay has been destroyed
self.assertRaises(exception.BayNotFound,
objects.Bay.get, self.context, self.bay.uuid)
class TestBayK8sHeatSwarm(base.TestCase):
def setUp(self):
super(TestBayK8sHeatSwarm, self).setUp()
self.baymodel_dict = {
'image_id': 'image_id',
'flavor_id': 'flavor_id',
'keypair_id': 'keypair_id',
'dns_nameserver': 'dns_nameserver',
'external_network_id': 'external_network_id',
'fixed_network': '10.2.0.0/22'
}
self.bay_dict = {
'id': 1,
'uuid': 'some_uuid',
'baymodel_id': 'xx-xx-xx-xx',
'name': 'bay1',
'stack_id': 'xx-xx-xx-xx',
'api_address': '172.17.2.3',
'node_addresses': ['172.17.2.4'],
'node_count': 1,
'discovery_url': 'token://39987da72f8386e0d0225ae8929e7cb4',
}
@patch('magnum.objects.BayModel.get_by_uuid')
def test_extract_template_definition_all_values(self,
mock_objects_baymodel_get_by_uuid):
cfg.CONF.set_override('cluster_coe', 'swarm', group='k8s_heat')
baymodel = objects.BayModel(self.context, **self.baymodel_dict)
mock_objects_baymodel_get_by_uuid.return_value = baymodel
bay = objects.Bay(self.context, **self.bay_dict)
(template_path,
definition) = bay_k8s_heat._extract_template_definition(self.context,
bay)
expected = {
'ssh_key_name': 'keypair_id',
'external_network_id': 'external_network_id',
'dns_nameserver': 'dns_nameserver',
'server_image': 'image_id',
'server_flavor': 'flavor_id',
'number_of_nodes': '1',
'fixed_network_cidr': '10.2.0.0/22',
'discovery_url': 'token://39987da72f8386e0d0225ae8929e7cb4'
}
self.assertEqual(expected, definition)
@patch('magnum.objects.BayModel.get_by_uuid')
def test_extract_template_definition_only_required(self,
mock_objects_baymodel_get_by_uuid):
cfg.CONF.set_override('cluster_coe', 'swarm', group='k8s_heat')
cfg.CONF.set_override('public_swarm_discovery', False, group='bay')
cfg.CONF.set_override('swarm_discovery_url_format',
'test_discovery', group='bay')
not_required = ['image_id', 'flavor_id', 'dns_nameserver',
'fixed_network']
for key in not_required:
self.baymodel_dict[key] = None
self.bay_dict['discovery_url'] = None
baymodel = objects.BayModel(self.context, **self.baymodel_dict)
mock_objects_baymodel_get_by_uuid.return_value = baymodel
bay = objects.Bay(self.context, **self.bay_dict)
(template_path,
definition) = bay_k8s_heat._extract_template_definition(self.context,
bay)
expected = {
'ssh_key_name': 'keypair_id',
'external_network_id': 'external_network_id',
'number_of_nodes': '1',
'discovery_url': 'test_discovery'
}
self.assertEqual(expected, definition)

View File

@ -48,6 +48,13 @@ class TemplateDefinitionTestCase(base.TestCase):
self.assertIsInstance(definition,
tdef.CoreOSK8sTemplateDefinition)
def test_get_vm_atomic_swarm_definition(self):
definition = tdef.TemplateDefinition.get_template_definition('vm',
'fedora-atomic', 'swarm')
self.assertIsInstance(definition,
tdef.AtomicSwarmTemplateDefinition)
def test_get_definition_not_supported(self):
self.assertRaises(exception.BayTypeNotSupported,
tdef.TemplateDefinition.get_template_definition,
@ -69,3 +76,61 @@ class TemplateDefinitionTestCase(base.TestCase):
self.assertRaises(exception.RequiredParameterNotProvided,
param.set_param, {}, mock_baymodel, None)
@mock.patch('requests.post')
def test_swarm_discovery_url_public_token(self, mock_post):
mock_resp = mock.MagicMock()
mock_resp.text = 'some_token'
mock_post.return_value = mock_resp
mock_bay = mock.MagicMock()
mock_bay.discovery_url = None
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual('token://some_token', actual_url)
def test_swarm_discovery_url_format_bay_id(self):
cfg.CONF.set_override('public_swarm_discovery', False, group='bay')
cfg.CONF.set_override('swarm_discovery_url_format',
'etcd://test.com/bay-%(bay_id)s', group='bay')
mock_bay = mock.MagicMock()
mock_bay.discovery_url = None
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual('etcd://test.com/bay-1', actual_url)
def test_swarm_discovery_url_format_bay_uuid(self):
cfg.CONF.set_override('public_swarm_discovery', False, group='bay')
cfg.CONF.set_override('swarm_discovery_url_format',
'etcd://test.com/bay-%(bay_uuid)s', group='bay')
mock_bay = mock.MagicMock()
mock_bay.discovery_url = None
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual('etcd://test.com/bay-some_uuid', actual_url)
def test_swarm_discovery_url_from_bay(self):
mock_bay = mock.MagicMock()
mock_bay.discovery_url = 'token://some_token'
mock_bay.id = 1
mock_bay.uuid = 'some_uuid'
swarm_def = tdef.AtomicSwarmTemplateDefinition()
actual_url = swarm_def.get_discovery_url(mock_bay)
self.assertEqual(mock_bay.discovery_url, actual_url)

View File

@ -53,6 +53,7 @@ def get_test_bay(**kw):
'id': kw.get('id', 42),
'uuid': kw.get('uuid', '5d12f6fd-a196-4bf0-ae4c-1f639a523a52'),
'name': kw.get('name', 'bay1'),
'discovery_url': kw.get('discovery_url', None),
'project_id': kw.get('project_id', 'fake_project'),
'user_id': kw.get('user_id', 'fake_user'),
'baymodel_id': kw.get('baymodel_id',

View File

@ -56,6 +56,7 @@ oslo.config.opts =
magnum.template_definitions =
magnum_vm_atomic_k8s = magnum.conductor.template_definition:AtomicK8sTemplateDefinition
magnum_vm_coreos_k8s = magnum.conductor.template_definition:CoreOSK8sTemplateDefinition
magnum_vm_atomic_swarm = magnum.conductor.template_definition:AtomicSwarmTemplateDefinition
[wheel]
universal = 1