From 4080d4a84c4220153e629fd8ac973e6a8febf5d3 Mon Sep 17 00:00:00 2001 From: Murali Allada Date: Mon, 23 May 2016 09:27:27 -0500 Subject: [PATCH] Bay driver implementation Moved all the swarm templates and template_definition code to the magnum/drivers folder. Moved base template_definition classes to drivers/common folder Change-Id: Ieff57f0f47835c35d9f17c3d7d1b7e6a40907462 Partially-Implements: blueprint bay-drivers Co-Authored-by: Spyros Trigazis --- magnum/cmd/template_manage.py | 2 +- magnum/conductor/handlers/bay_conductor.py | 2 +- magnum/conductor/template_definition.py | 498 +----------------- magnum/drivers/__init__.py | 0 magnum/drivers/common/__init__.py | 0 magnum/drivers/common/template_def.py | 452 ++++++++++++++++ .../swarm_fedora_atomic_v1/__init__.py | 0 .../swarm_fedora_atomic_v1/template_def.py | 118 +++++ .../swarm_fedora_atomic_v1/templates}/COPYING | 0 .../templates}/README.md | 0 .../templates/cluster.yaml} | 0 .../templates}/fragments/add-proxy.sh | 0 .../templates}/fragments/cfn-signal.sh | 0 .../fragments/configure-docker-registry.sh | 0 .../fragments/configure-docker-storage.sh | 0 .../templates}/fragments/configure-etcd.sh | 0 .../configure_docker_storage_driver_atomic.sh | 38 ++ .../templates}/fragments/disable-selinux.sh | 0 .../fragments/enable-docker-registry.sh | 0 .../templates}/fragments/enable-services.sh | 0 .../templates}/fragments/make-cert.py | 0 .../fragments/network-config-service.sh | 0 .../templates}/fragments/network-service.sh | 0 .../templates}/fragments/remove-docker-key.sh | 0 .../fragments/write-bay-failure-service.yaml | 0 .../fragments/write-docker-service.sh | 0 .../fragments/write-docker-socket.yaml | 0 .../fragments/write-heat-params-master.yaml | 0 .../fragments/write-heat-params-node.yaml | 0 .../fragments/write-network-config.sh | 0 .../fragments/write-swarm-agent-service.sh | 0 .../fragments/write-swarm-master-service.sh | 0 .../templates}/swarmmaster.yaml | 3 +- .../templates}/swarmnode.yaml | 2 +- .../drivers/swarm_fedora_atomic_v1/version.py | 16 + magnum/tests/functional/k8s/test_templates.py | 2 +- .../conductor/test_template_definition.py | 69 +-- setup.cfg | 2 +- 38 files changed, 670 insertions(+), 534 deletions(-) create mode 100644 magnum/drivers/__init__.py create mode 100644 magnum/drivers/common/__init__.py create mode 100644 magnum/drivers/common/template_def.py create mode 100644 magnum/drivers/swarm_fedora_atomic_v1/__init__.py create mode 100644 magnum/drivers/swarm_fedora_atomic_v1/template_def.py rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/COPYING (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/README.md (100%) rename magnum/{templates/swarm/swarmcluster.yaml => drivers/swarm_fedora_atomic_v1/templates/cluster.yaml} (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/add-proxy.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/cfn-signal.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/configure-docker-registry.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/configure-docker-storage.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/configure-etcd.sh (100%) create mode 100644 magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure_docker_storage_driver_atomic.sh rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/disable-selinux.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/enable-docker-registry.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/enable-services.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/make-cert.py (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/network-config-service.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/network-service.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/remove-docker-key.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-bay-failure-service.yaml (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-docker-service.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-docker-socket.yaml (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-heat-params-master.yaml (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-heat-params-node.yaml (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-network-config.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-swarm-agent-service.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/fragments/write-swarm-master-service.sh (100%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/swarmmaster.yaml (99%) rename magnum/{templates/swarm => drivers/swarm_fedora_atomic_v1/templates}/swarmnode.yaml (98%) create mode 100644 magnum/drivers/swarm_fedora_atomic_v1/version.py diff --git a/magnum/cmd/template_manage.py b/magnum/cmd/template_manage.py index f2cfa0140f..9ef6f0a27f 100644 --- a/magnum/cmd/template_manage.py +++ b/magnum/cmd/template_manage.py @@ -19,7 +19,7 @@ from cliff import commandmanager from cliff import lister from oslo_config import cfg -from magnum.conductor import template_definition as tdef +from magnum.drivers.common import template_def as tdef from magnum import version CONF = cfg.CONF diff --git a/magnum/conductor/handlers/bay_conductor.py b/magnum/conductor/handlers/bay_conductor.py index b21d5504ff..ec8546c6e1 100644 --- a/magnum/conductor/handlers/bay_conductor.py +++ b/magnum/conductor/handlers/bay_conductor.py @@ -29,8 +29,8 @@ from magnum.common import short_id from magnum.conductor.handlers.common import cert_manager from magnum.conductor.handlers.common import trust_manager from magnum.conductor import scale_manager -from magnum.conductor.template_definition import TemplateDefinition as TDef from magnum.conductor import utils as conductor_utils +from magnum.drivers.common.template_def import TemplateDefinition as TDef from magnum.i18n import _ from magnum.i18n import _LE from magnum.i18n import _LI diff --git a/magnum/conductor/template_definition.py b/magnum/conductor/template_definition.py index d72c04f018..530783572e 100644 --- a/magnum/conductor/template_definition.py +++ b/magnum/conductor/template_definition.py @@ -11,28 +11,18 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -import abc -import ast - from oslo_config import cfg from oslo_log import log as logging -from pkg_resources import iter_entry_points -import requests -from requests import exceptions as req_exceptions -import six -from magnum.common import clients -from magnum.common import exception from magnum.common import paths +from magnum.drivers.common import template_def from magnum.i18n import _ -from magnum.i18n import _LW LOG = logging.getLogger(__name__) KUBE_SECURE_PORT = '6443' KUBE_INSECURE_PORT = '8080' -DOCKER_PORT = '2376' template_def_opts = [ cfg.StrOpt('k8s_atomic_template_path', @@ -50,11 +40,6 @@ template_def_opts = [ cfg.StrOpt('etcd_discovery_service_endpoint_format', default='https://discovery.etcd.io/new?size=%(size)d', help=_('Url for etcd public discovery endpoint.')), - cfg.StrOpt('swarm_atomic_template_path', - default=paths.basedir_def('templates/swarm/' - 'swarmcluster.yaml'), - help=_('Location of template to build a swarm ' - 'cluster on atomic.')), cfg.StrOpt('mesos_ubuntu_template_path', default=paths.basedir_def('templates/mesos/' 'mesoscluster.yaml'), @@ -80,385 +65,7 @@ CONF.register_opts(template_def_opts, group='bay') CONF.register_opts(docker_registry_opts, group='docker_registry') -class ParameterMapping(object): - """A mapping associating heat param and bay/baymodel attr. - - A ParameterMapping is an association of a Heat parameter name with - an attribute on a Bay, Baymodel, or both. - - In the case of both baymodel_attr and bay_attr being set, the Baymodel - will be checked first and then Bay if the attribute isn't set on the - Baymodel. - - Parameters can also be set as 'required'. If a required parameter - isn't set, a RequiredArgumentNotProvided exception will be raised. - """ - def __init__(self, heat_param, baymodel_attr=None, - bay_attr=None, required=False, - param_type=lambda x: x): - self.heat_param = heat_param - self.baymodel_attr = baymodel_attr - self.bay_attr = bay_attr - self.required = required - self.param_type = param_type - - def set_param(self, params, baymodel, bay): - value = None - - if (self.baymodel_attr and - getattr(baymodel, self.baymodel_attr, None) is not None): - value = getattr(baymodel, self.baymodel_attr) - elif (self.bay_attr and - getattr(bay, self.bay_attr, None) is not None): - value = getattr(bay, self.bay_attr) - elif self.required: - kwargs = dict(heat_param=self.heat_param) - raise exception.RequiredParameterNotProvided(**kwargs) - - if value is not None: - value = self.param_type(value) - params[self.heat_param] = value - - -class OutputMapping(object): - """A mapping associating heat outputs and bay attr. - - An OutputMapping is an association of a Heat output with a key - Magnum understands. - """ - - def __init__(self, heat_output, bay_attr=None): - self.bay_attr = bay_attr - self.heat_output = heat_output - - def set_output(self, stack, baymodel, bay): - if self.bay_attr is None: - return - - output_value = self.get_output_value(stack) - if output_value is not None: - setattr(bay, self.bay_attr, output_value) - - def matched(self, output_key): - return self.heat_output == output_key - - def get_output_value(self, stack): - for output in stack.to_dict().get('outputs', []): - if output['output_key'] == self.heat_output: - return output['output_value'] - - LOG.warning(_LW('stack does not have output_key %s'), self.heat_output) - return None - - -@six.add_metaclass(abc.ABCMeta) -class TemplateDefinition(object): - '''A mapping between Magnum objects and Heat templates. - - A TemplateDefinition is essentially a mapping between Magnum objects - and Heat templates. Each TemplateDefinition has a mapping of Heat - parameters. - ''' - definitions = None - provides = list() - - def __init__(self): - self.param_mappings = list() - self.output_mappings = list() - - @staticmethod - def load_entry_points(): - for entry_point in iter_entry_points('magnum.template_definitions'): - yield entry_point, entry_point.load(require=False) - - @classmethod - def get_template_definitions(cls): - '''Retrieves bay definitions from python entry_points. - - Example: - - With the following classes: - class TemplateDefinition1(TemplateDefinition): - provides = [ - ('server_type1', 'os1', 'coe1') - ] - - class TemplateDefinition2(TemplateDefinition): - provides = [ - ('server_type2', 'os2', 'coe2') - ] - - And the following entry_points: - - magnum.template_definitions = - template_name_1 = some.python.path:TemplateDefinition1 - template_name_2 = some.python.path:TemplateDefinition2 - - get_template_definitions will return: - { - (server_type1, os1, coe1): - {'template_name_1': TemplateDefinition1}, - (server_type2, os2, coe2): - {'template_name_2': TemplateDefinition2} - } - - :return: dict - ''' - - if not cls.definitions: - cls.definitions = dict() - for entry_point, def_class in cls.load_entry_points(): - for bay_type in def_class.provides: - bay_type_tuple = (bay_type['server_type'], - bay_type['os'], - bay_type['coe']) - providers = cls.definitions.setdefault(bay_type_tuple, - dict()) - providers[entry_point.name] = def_class - - return cls.definitions - - @classmethod - def get_template_definition(cls, server_type, os, coe): - '''Get enabled TemplateDefinitions. - - Returns the enabled TemplateDefinition class for the provided - bay_type. - - With the following classes: - class TemplateDefinition1(TemplateDefinition): - provides = [ - ('server_type1', 'os1', 'coe1') - ] - - class TemplateDefinition2(TemplateDefinition): - provides = [ - ('server_type2', 'os2', 'coe2') - ] - - And the following entry_points: - - magnum.template_definitions = - template_name_1 = some.python.path:TemplateDefinition1 - template_name_2 = some.python.path:TemplateDefinition2 - - get_template_name_1_definition('server_type2', 'os2', 'coe2') - will return: TemplateDefinition2 - - :param server_type: The server_type the bay definition - will build on - :param os: The operation system the bay definition will build on - :param coe: The Container Orchestration Environment the bay will - produce - - :return: class - ''' - - definition_map = cls.get_template_definitions() - bay_type = (server_type, os, coe) - - if bay_type not in definition_map: - raise exception.BayTypeNotSupported( - server_type=server_type, - os=os, - coe=coe) - type_definitions = definition_map[bay_type] - - for name in cfg.CONF.bay.enabled_definitions: - if name in type_definitions: - return type_definitions[name]() - - raise exception.BayTypeNotEnabled( - server_type=server_type, os=os, coe=coe) - - def add_parameter(self, *args, **kwargs): - param = ParameterMapping(*args, **kwargs) - self.param_mappings.append(param) - - def add_output(self, *args, **kwargs): - mapping_type = kwargs.pop('mapping_type', OutputMapping) - output = mapping_type(*args, **kwargs) - self.output_mappings.append(output) - - def get_output(self, *args, **kwargs): - for output in self.output_mappings: - if output.matched(*args, **kwargs): - return output - - return None - - def get_params(self, context, baymodel, bay, **kwargs): - """Pulls template parameters from Baymodel and/or Bay. - - :param context: Context to pull template parameters for - :param baymodel: Baymodel to pull template parameters from - :param bay: Bay to pull template parameters from - :param extra_params: Any extra params to be provided to the template - - :return: dict of template parameters - """ - template_params = dict() - - for mapping in self.param_mappings: - mapping.set_param(template_params, baymodel, bay) - - if 'extra_params' in kwargs: - template_params.update(kwargs.get('extra_params')) - - return template_params - - def get_env_files(self, baymodel): - """Collects stack environment files based upon Baymodel attributes. - - Base implementation returns no files (empty list). Meant to be - overridden by subclasses. - - :param baymodel: Baymodel to collect environment files for - - :return: list of relative paths to environment files - """ - return [] - - def get_heat_param(self, bay_attr=None, baymodel_attr=None): - """Returns stack param name. - - Return stack param name using bay and baymodel attributes - :param bay_attr bay attribute from which it maps to stack attribute - :param baymodel_attr baymodel attribute from which it maps - to stack attribute - - :return stack parameter name or None - """ - for mapping in self.param_mappings: - if (mapping.bay_attr == bay_attr and - mapping.baymodel_attr == baymodel_attr): - return mapping.heat_param - - return None - - def update_outputs(self, stack, baymodel, bay): - for output in self.output_mappings: - output.set_output(stack, baymodel, bay) - - @abc.abstractproperty - def template_path(self): - pass - - def extract_definition(self, context, baymodel, bay, **kwargs): - return (self.template_path, - self.get_params(context, baymodel, bay, **kwargs), - self.get_env_files(baymodel)) - - -class BaseTemplateDefinition(TemplateDefinition): - def __init__(self): - super(BaseTemplateDefinition, self).__init__() - self._osc = None - - self.add_parameter('ssh_key_name', - baymodel_attr='keypair_id', - required=True) - self.add_parameter('server_image', - baymodel_attr='image_id') - self.add_parameter('dns_nameserver', - baymodel_attr='dns_nameserver') - self.add_parameter('http_proxy', - baymodel_attr='http_proxy') - self.add_parameter('https_proxy', - baymodel_attr='https_proxy') - self.add_parameter('no_proxy', - baymodel_attr='no_proxy') - self.add_parameter('number_of_masters', - bay_attr='master_count') - - @abc.abstractproperty - def template_path(self): - pass - - def get_osc(self, context): - if not self._osc: - self._osc = clients.OpenStackClients(context) - return self._osc - - def get_params(self, context, baymodel, bay, **kwargs): - osc = self.get_osc(context) - - extra_params = kwargs.pop('extra_params', {}) - extra_params['trustee_domain_id'] = osc.keystone().trustee_domain_id - extra_params['trustee_user_id'] = bay.trustee_user_id - extra_params['trustee_username'] = bay.trustee_username - extra_params['trustee_password'] = bay.trustee_password - extra_params['trust_id'] = bay.trust_id - extra_params['auth_url'] = context.auth_url - - return super(BaseTemplateDefinition, - self).get_params(context, baymodel, bay, - extra_params=extra_params, - **kwargs) - - def validate_discovery_url(self, discovery_url, expect_size): - url = str(discovery_url) - if url[len(url)-1] == '/': - url += '_config/size' - else: - url += '/_config/size' - - try: - result = requests.get(url).text - except req_exceptions.RequestException as err: - LOG.error(six.text_type(err)) - raise exception.GetClusterSizeFailed( - discovery_url=discovery_url) - - try: - result = ast.literal_eval(result) - except (ValueError, SyntaxError): - raise exception.InvalidBayDiscoveryURL( - discovery_url=discovery_url) - - node_value = result.get('node', None) - if node_value is None: - raise exception.InvalidBayDiscoveryURL( - discovery_url=discovery_url) - - value = node_value.get('value', None) - if value is None: - raise exception.InvalidBayDiscoveryURL( - discovery_url=discovery_url) - elif int(value) != expect_size: - raise exception.InvalidClusterSize( - expect_size=expect_size, - size=int(value), - discovery_url=discovery_url) - - def get_discovery_url(self, bay): - if hasattr(bay, 'discovery_url') and bay.discovery_url: - if getattr(bay, 'master_count', None) is not None: - self.validate_discovery_url(bay.discovery_url, - bay.master_count) - else: - self.validate_discovery_url(bay.discovery_url, 1) - discovery_url = bay.discovery_url - else: - discovery_endpoint = ( - cfg.CONF.bay.etcd_discovery_service_endpoint_format % - {'size': bay.master_count}) - try: - discovery_url = requests.get(discovery_endpoint).text - except req_exceptions.RequestException as err: - LOG.error(six.text_type(err)) - raise exception.GetDiscoveryUrlFailed( - discovery_endpoint=discovery_endpoint) - if not discovery_url: - raise exception.InvalidDiscoveryURL( - discovery_url=discovery_url, - discovery_endpoint=discovery_endpoint) - else: - bay.discovery_url = discovery_url - return discovery_url - - -class K8sApiAddressOutputMapping(OutputMapping): +class K8sApiAddressOutputMapping(template_def.OutputMapping): def set_output(self, stack, baymodel, bay): if self.bay_attr is None: @@ -482,28 +89,7 @@ class K8sApiAddressOutputMapping(OutputMapping): setattr(bay, self.bay_attr, value) -class SwarmApiAddressOutputMapping(OutputMapping): - - def set_output(self, stack, baymodel, bay): - if self.bay_attr is None: - return - - output_value = self.get_output_value(stack) - if output_value is not None: - protocol = 'https' - if baymodel.tls_disabled: - protocol = 'tcp' - - params = { - 'protocol': protocol, - 'address': output_value, - 'port': DOCKER_PORT, - } - value = "%(protocol)s://%(address)s:%(port)s" % params - setattr(bay, self.bay_attr, value) - - -class K8sTemplateDefinition(BaseTemplateDefinition): +class K8sTemplateDefinition(template_def.BaseTemplateDefinition): """Base Kubernetes template.""" def __init__(self): @@ -622,83 +208,7 @@ class CoreOSK8sTemplateDefinition(K8sTemplateDefinition): return cfg.CONF.bay.k8s_coreos_template_path -class AtomicSwarmTemplateDefinition(BaseTemplateDefinition): - """Docker swarm template for a Fedora Atomic VM.""" - - provides = [ - {'server_type': 'vm', 'os': 'fedora-atomic', 'coe': 'swarm'}, - ] - - def __init__(self): - super(AtomicSwarmTemplateDefinition, self).__init__() - self.add_parameter('bay_uuid', - bay_attr='uuid', - param_type=str) - self.add_parameter('number_of_nodes', - bay_attr='node_count') - self.add_parameter('master_flavor', - baymodel_attr='master_flavor_id') - self.add_parameter('node_flavor', - baymodel_attr='flavor_id') - self.add_parameter('docker_volume_size', - baymodel_attr='docker_volume_size') - self.add_parameter('external_network', - baymodel_attr='external_network_id', - required=True) - self.add_parameter('network_driver', - baymodel_attr='network_driver') - self.add_parameter('tls_disabled', - baymodel_attr='tls_disabled', - required=True) - self.add_parameter('registry_enabled', - baymodel_attr='registry_enabled') - self.add_parameter('docker_storage_driver', - baymodel_attr='docker_storage_driver') - self.add_output('api_address', - bay_attr='api_address', - mapping_type=SwarmApiAddressOutputMapping) - self.add_output('swarm_master_private', - bay_attr=None) - self.add_output('swarm_masters', - bay_attr='master_addresses') - self.add_output('swarm_nodes_private', - bay_attr=None) - self.add_output('swarm_nodes', - bay_attr='node_addresses') - self.add_output('discovery_url', - bay_attr='discovery_url') - - def get_params(self, context, baymodel, bay, **kwargs): - extra_params = kwargs.pop('extra_params', {}) - extra_params['discovery_url'] = self.get_discovery_url(bay) - # HACK(apmelton) - This uses the user's bearer token, ideally - # it should be replaced with an actual trust token with only - # access to do what the template needs it to do. - osc = self.get_osc(context) - extra_params['magnum_url'] = osc.magnum_url() - - label_list = ['flannel_network_cidr', 'flannel_backend', - 'flannel_network_subnetlen'] - - for label in label_list: - extra_params[label] = baymodel.labels.get(label) - - if baymodel.registry_enabled: - extra_params['swift_region'] = CONF.docker_registry.swift_region - extra_params['registry_container'] = ( - CONF.docker_registry.swift_registry_container) - - return super(AtomicSwarmTemplateDefinition, - self).get_params(context, baymodel, bay, - extra_params=extra_params, - **kwargs) - - @property - def template_path(self): - return cfg.CONF.bay.swarm_atomic_template_path - - -class UbuntuMesosTemplateDefinition(BaseTemplateDefinition): +class UbuntuMesosTemplateDefinition(template_def.BaseTemplateDefinition): """Mesos template for Ubuntu VM.""" provides = [ diff --git a/magnum/drivers/__init__.py b/magnum/drivers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/magnum/drivers/common/__init__.py b/magnum/drivers/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/magnum/drivers/common/template_def.py b/magnum/drivers/common/template_def.py new file mode 100644 index 0000000000..0666a9eb86 --- /dev/null +++ b/magnum/drivers/common/template_def.py @@ -0,0 +1,452 @@ +# Copyright 2016 Rackspace Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import abc +import ast + +from oslo_config import cfg +from oslo_log import log as logging +from pkg_resources import iter_entry_points +import requests +import six + +from magnum.common import clients +from magnum.common import exception +from magnum.common import paths +from magnum.i18n import _ +from magnum.i18n import _LW + +from requests import exceptions as req_exceptions + + +LOG = logging.getLogger(__name__) + + +template_def_opts = [ + cfg.StrOpt('k8s_atomic_template_path', + default=paths.basedir_def('templates/kubernetes/' + 'kubecluster.yaml'), + deprecated_name='template_path', + deprecated_group='bay_heat', + help=_( + 'Location of template to build a k8s cluster on atomic.')), + cfg.StrOpt('k8s_coreos_template_path', + default=paths.basedir_def('templates/kubernetes/' + 'kubecluster-coreos.yaml'), + help=_( + 'Location of template to build a k8s cluster on CoreOS.')), + cfg.StrOpt('etcd_discovery_service_endpoint_format', + default='https://discovery.etcd.io/new?size=%(size)d', + help=_('Url for etcd public discovery endpoint.')), + cfg.StrOpt('mesos_ubuntu_template_path', + default=paths.basedir_def('templates/mesos/' + 'mesoscluster.yaml'), + help=_('Location of template to build a Mesos cluster ' + 'on Ubuntu.')), + cfg.ListOpt('enabled_definitions', + default=['magnum_vm_atomic_k8s', 'magnum_vm_coreos_k8s', + 'magnum_vm_atomic_swarm', 'magnum_vm_ubuntu_mesos'], + help=_('Enabled bay definition entry points.')), +] + +docker_registry_opts = [ + cfg.StrOpt('swift_region', + help=_('Region name of Swift')), + cfg.StrOpt('swift_registry_container', + default='docker_registry', + help=_('Name of the container in Swift which docker registry ' + 'stores images in')) +] + +CONF = cfg.CONF +CONF.register_opts(template_def_opts, group='bay') +CONF.register_opts(docker_registry_opts, group='docker_registry') +CONF.import_opt('trustee_domain_id', 'magnum.common.keystone', group='trust') + + +class ParameterMapping(object): + """A mapping associating heat param and bay/baymodel attr. + + A ParameterMapping is an association of a Heat parameter name with + an attribute on a Bay, Baymodel, or both. + + In the case of both baymodel_attr and bay_attr being set, the Baymodel + will be checked first and then Bay if the attribute isn't set on the + Baymodel. + + Parameters can also be set as 'required'. If a required parameter + isn't set, a RequiredArgumentNotProvided exception will be raised. + """ + def __init__(self, heat_param, baymodel_attr=None, + bay_attr=None, required=False, + param_type=lambda x: x): + self.heat_param = heat_param + self.baymodel_attr = baymodel_attr + self.bay_attr = bay_attr + self.required = required + self.param_type = param_type + + def set_param(self, params, baymodel, bay): + value = None + + if (self.baymodel_attr and + getattr(baymodel, self.baymodel_attr, None) is not None): + value = getattr(baymodel, self.baymodel_attr) + elif (self.bay_attr and + getattr(bay, self.bay_attr, None) is not None): + value = getattr(bay, self.bay_attr) + elif self.required: + kwargs = dict(heat_param=self.heat_param) + raise exception.RequiredParameterNotProvided(**kwargs) + + if value is not None: + value = self.param_type(value) + params[self.heat_param] = value + + +class OutputMapping(object): + """A mapping associating heat outputs and bay attr. + + An OutputMapping is an association of a Heat output with a key + Magnum understands. + """ + + def __init__(self, heat_output, bay_attr=None): + self.bay_attr = bay_attr + self.heat_output = heat_output + + def set_output(self, stack, baymodel, bay): + if self.bay_attr is None: + return + + output_value = self.get_output_value(stack) + if output_value is not None: + setattr(bay, self.bay_attr, output_value) + + def matched(self, output_key): + return self.heat_output == output_key + + def get_output_value(self, stack): + for output in stack.to_dict().get('outputs', []): + if output['output_key'] == self.heat_output: + return output['output_value'] + + LOG.warning(_LW('stack does not have output_key %s'), self.heat_output) + return None + + +@six.add_metaclass(abc.ABCMeta) +class TemplateDefinition(object): + '''A mapping between Magnum objects and Heat templates. + + A TemplateDefinition is essentially a mapping between Magnum objects + and Heat templates. Each TemplateDefinition has a mapping of Heat + parameters. + ''' + definitions = None + provides = list() + + def __init__(self): + self.param_mappings = list() + self.output_mappings = list() + + @staticmethod + def load_entry_points(): + for entry_point in iter_entry_points('magnum.template_definitions'): + yield entry_point, entry_point.load(require=False) + + @classmethod + def get_template_definitions(cls): + '''Retrieves bay definitions from python entry_points. + + Example: + + With the following classes: + class TemplateDefinition1(TemplateDefinition): + provides = [ + ('server_type1', 'os1', 'coe1') + ] + + class TemplateDefinition2(TemplateDefinition): + provides = [ + ('server_type2', 'os2', 'coe2') + ] + + And the following entry_points: + + magnum.template_definitions = + template_name_1 = some.python.path:TemplateDefinition1 + template_name_2 = some.python.path:TemplateDefinition2 + + get_template_definitions will return: + { + (server_type1, os1, coe1): + {'template_name_1': TemplateDefinition1}, + (server_type2, os2, coe2): + {'template_name_2': TemplateDefinition2} + } + + :return: dict + ''' + + if not cls.definitions: + cls.definitions = dict() + for entry_point, def_class in cls.load_entry_points(): + for bay_type in def_class.provides: + bay_type_tuple = (bay_type['server_type'], + bay_type['os'], + bay_type['coe']) + providers = cls.definitions.setdefault(bay_type_tuple, + dict()) + providers[entry_point.name] = def_class + + return cls.definitions + + @classmethod + def get_template_definition(cls, server_type, os, coe): + '''Get enabled TemplateDefinitions. + + Returns the enabled TemplateDefinition class for the provided + bay_type. + + With the following classes: + class TemplateDefinition1(TemplateDefinition): + provides = [ + ('server_type1', 'os1', 'coe1') + ] + + class TemplateDefinition2(TemplateDefinition): + provides = [ + ('server_type2', 'os2', 'coe2') + ] + + And the following entry_points: + + magnum.template_definitions = + template_name_1 = some.python.path:TemplateDefinition1 + template_name_2 = some.python.path:TemplateDefinition2 + + get_template_name_1_definition('server_type2', 'os2', 'coe2') + will return: TemplateDefinition2 + + :param server_type: The server_type the bay definition + will build on + :param os: The operation system the bay definition will build on + :param coe: The Container Orchestration Environment the bay will + produce + + :return: class + ''' + + definition_map = cls.get_template_definitions() + bay_type = (server_type, os, coe) + + if bay_type not in definition_map: + raise exception.BayTypeNotSupported( + server_type=server_type, + os=os, + coe=coe) + type_definitions = definition_map[bay_type] + + for name in cfg.CONF.bay.enabled_definitions: + if name in type_definitions: + return type_definitions[name]() + + raise exception.BayTypeNotEnabled( + server_type=server_type, os=os, coe=coe) + + def add_parameter(self, *args, **kwargs): + param = ParameterMapping(*args, **kwargs) + self.param_mappings.append(param) + + def add_output(self, *args, **kwargs): + mapping_type = kwargs.pop('mapping_type', OutputMapping) + output = mapping_type(*args, **kwargs) + self.output_mappings.append(output) + + def get_output(self, *args, **kwargs): + for output in self.output_mappings: + if output.matched(*args, **kwargs): + return output + + return None + + def get_params(self, context, baymodel, bay, **kwargs): + """Pulls template parameters from Baymodel and/or Bay. + + :param context: Context to pull template parameters for + :param baymodel: Baymodel to pull template parameters from + :param bay: Bay to pull template parameters from + :param extra_params: Any extra params to be provided to the template + + :return: dict of template parameters + """ + template_params = dict() + + for mapping in self.param_mappings: + mapping.set_param(template_params, baymodel, bay) + + if 'extra_params' in kwargs: + template_params.update(kwargs.get('extra_params')) + + return template_params + + def get_env_files(self, baymodel): + """Collects stack environment files based upon Baymodel attributes. + + Base implementation returns no files (empty list). Meant to be + overridden by subclasses. + + :param baymodel: Baymodel to collect environment files for + + :return: list of relative paths to environment files + """ + return [] + + def get_heat_param(self, bay_attr=None, baymodel_attr=None): + """Returns stack param name. + + Return stack param name using bay and baymodel attributes + :param bay_attr bay attribute from which it maps to stack attribute + :param baymodel_attr baymodel attribute from which it maps + to stack attribute + + :return stack parameter name or None + """ + for mapping in self.param_mappings: + if (mapping.bay_attr == bay_attr and + mapping.baymodel_attr == baymodel_attr): + return mapping.heat_param + + return None + + def update_outputs(self, stack, baymodel, bay): + for output in self.output_mappings: + output.set_output(stack, baymodel, bay) + + @abc.abstractproperty + def template_path(self): + pass + + def extract_definition(self, context, baymodel, bay, **kwargs): + return (self.template_path, + self.get_params(context, baymodel, bay, **kwargs), + self.get_env_files(baymodel)) + + +class BaseTemplateDefinition(TemplateDefinition): + def __init__(self): + super(BaseTemplateDefinition, self).__init__() + self._osc = None + + self.add_parameter('ssh_key_name', + baymodel_attr='keypair_id', + required=True) + self.add_parameter('server_image', + baymodel_attr='image_id') + self.add_parameter('dns_nameserver', + baymodel_attr='dns_nameserver') + self.add_parameter('http_proxy', + baymodel_attr='http_proxy') + self.add_parameter('https_proxy', + baymodel_attr='https_proxy') + self.add_parameter('no_proxy', + baymodel_attr='no_proxy') + self.add_parameter('number_of_masters', + bay_attr='master_count') + + @abc.abstractproperty + def template_path(self): + pass + + def get_osc(self, context): + if not self._osc: + self._osc = clients.OpenStackClients(context) + return self._osc + + def get_params(self, context, baymodel, bay, **kwargs): + osc = self.get_osc(context) + + extra_params = kwargs.pop('extra_params', {}) + extra_params['trustee_domain_id'] = osc.keystone().trustee_domain_id + extra_params['trustee_user_id'] = bay.trustee_user_id + extra_params['trustee_username'] = bay.trustee_username + extra_params['trustee_password'] = bay.trustee_password + extra_params['trust_id'] = bay.trust_id + extra_params['auth_url'] = context.auth_url + + return super(BaseTemplateDefinition, + self).get_params(context, baymodel, bay, + extra_params=extra_params, + **kwargs) + + def validate_discovery_url(self, discovery_url, expect_size): + url = str(discovery_url) + if url[len(url)-1] == '/': + url += '_config/size' + else: + url += '/_config/size' + + try: + result = requests.get(url).text + except req_exceptions.RequestException as err: + LOG.error(six.text_type(err)) + raise exception.GetClusterSizeFailed( + discovery_url=discovery_url) + + try: + result = ast.literal_eval(result) + except (ValueError, SyntaxError): + raise exception.InvalidBayDiscoveryURL( + discovery_url=discovery_url) + + node_value = result.get('node', None) + if node_value is None: + raise exception.InvalidBayDiscoveryURL( + discovery_url=discovery_url) + + value = node_value.get('value', None) + if value is None: + raise exception.InvalidBayDiscoveryURL( + discovery_url=discovery_url) + elif int(value) != expect_size: + raise exception.InvalidClusterSize( + expect_size=expect_size, + size=int(value), + discovery_url=discovery_url) + + def get_discovery_url(self, bay): + if hasattr(bay, 'discovery_url') and bay.discovery_url: + if getattr(bay, 'master_count', None) is not None: + self.validate_discovery_url(bay.discovery_url, + bay.master_count) + else: + self.validate_discovery_url(bay.discovery_url, 1) + discovery_url = bay.discovery_url + else: + discovery_endpoint = ( + cfg.CONF.bay.etcd_discovery_service_endpoint_format % + {'size': bay.master_count}) + try: + discovery_url = requests.get(discovery_endpoint).text + except req_exceptions.RequestException as err: + LOG.error(six.text_type(err)) + raise exception.GetDiscoveryUrlFailed( + discovery_endpoint=discovery_endpoint) + if not discovery_url: + raise exception.InvalidDiscoveryURL( + discovery_url=discovery_url, + discovery_endpoint=discovery_endpoint) + else: + bay.discovery_url = discovery_url + return discovery_url diff --git a/magnum/drivers/swarm_fedora_atomic_v1/__init__.py b/magnum/drivers/swarm_fedora_atomic_v1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/magnum/drivers/swarm_fedora_atomic_v1/template_def.py b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py new file mode 100644 index 0000000000..90a6fed462 --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/template_def.py @@ -0,0 +1,118 @@ +# Copyright 2016 Rackspace Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import os + +from magnum.drivers.common import template_def +from oslo_config import cfg + +CONF = cfg.CONF +DOCKER_PORT = '2376' + + +class SwarmApiAddressOutputMapping(template_def.OutputMapping): + + def set_output(self, stack, baymodel, bay): + if self.bay_attr is None: + return + + output_value = self.get_output_value(stack) + if output_value is not None: + protocol = 'https' + if baymodel.tls_disabled: + protocol = 'tcp' + + params = { + 'protocol': protocol, + 'address': output_value, + 'port': DOCKER_PORT, + } + value = "%(protocol)s://%(address)s:%(port)s" % params + setattr(bay, self.bay_attr, value) + + +class AtomicSwarmTemplateDefinition(template_def.BaseTemplateDefinition): + """Docker swarm template for a Fedora Atomic VM.""" + + provides = [ + {'server_type': 'vm', 'os': 'fedora-atomic', 'coe': 'swarm'}, + ] + + def __init__(self): + super(AtomicSwarmTemplateDefinition, self).__init__() + self.add_parameter('bay_uuid', + bay_attr='uuid', + param_type=str) + self.add_parameter('number_of_nodes', + bay_attr='node_count') + self.add_parameter('master_flavor', + baymodel_attr='master_flavor_id') + self.add_parameter('node_flavor', + baymodel_attr='flavor_id') + self.add_parameter('docker_volume_size', + baymodel_attr='docker_volume_size') + self.add_parameter('external_network', + baymodel_attr='external_network_id', + required=True) + self.add_parameter('network_driver', + baymodel_attr='network_driver') + self.add_parameter('tls_disabled', + baymodel_attr='tls_disabled', + required=True) + self.add_parameter('registry_enabled', + baymodel_attr='registry_enabled') + self.add_parameter('docker_storage_driver', + baymodel_attr='docker_storage_driver') + self.add_output('api_address', + bay_attr='api_address', + mapping_type=SwarmApiAddressOutputMapping) + self.add_output('swarm_master_private', + bay_attr=None) + self.add_output('swarm_masters', + bay_attr='master_addresses') + self.add_output('swarm_nodes_private', + bay_attr=None) + self.add_output('swarm_nodes', + bay_attr='node_addresses') + self.add_output('discovery_url', + bay_attr='discovery_url') + + def get_params(self, context, baymodel, bay, **kwargs): + extra_params = kwargs.pop('extra_params', {}) + extra_params['discovery_url'] = self.get_discovery_url(bay) + # HACK(apmelton) - This uses the user's bearer token, ideally + # it should be replaced with an actual trust token with only + # access to do what the template needs it to do. + osc = self.get_osc(context) + extra_params['magnum_url'] = osc.magnum_url() + + label_list = ['flannel_network_cidr', 'flannel_backend', + 'flannel_network_subnetlen'] + + for label in label_list: + extra_params[label] = baymodel.labels.get(label) + + if baymodel.registry_enabled: + extra_params['swift_region'] = CONF.docker_registry.swift_region + extra_params['registry_container'] = ( + CONF.docker_registry.swift_registry_container) + + return super(AtomicSwarmTemplateDefinition, + self).get_params(context, baymodel, bay, + extra_params=extra_params, + **kwargs) + + @property + def template_path(self): + return os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'templates/cluster.yaml') diff --git a/magnum/templates/swarm/COPYING b/magnum/drivers/swarm_fedora_atomic_v1/templates/COPYING similarity index 100% rename from magnum/templates/swarm/COPYING rename to magnum/drivers/swarm_fedora_atomic_v1/templates/COPYING diff --git a/magnum/templates/swarm/README.md b/magnum/drivers/swarm_fedora_atomic_v1/templates/README.md similarity index 100% rename from magnum/templates/swarm/README.md rename to magnum/drivers/swarm_fedora_atomic_v1/templates/README.md diff --git a/magnum/templates/swarm/swarmcluster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml similarity index 100% rename from magnum/templates/swarm/swarmcluster.yaml rename to magnum/drivers/swarm_fedora_atomic_v1/templates/cluster.yaml diff --git a/magnum/templates/swarm/fragments/add-proxy.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/add-proxy.sh similarity index 100% rename from magnum/templates/swarm/fragments/add-proxy.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/add-proxy.sh diff --git a/magnum/templates/swarm/fragments/cfn-signal.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/cfn-signal.sh similarity index 100% rename from magnum/templates/swarm/fragments/cfn-signal.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/cfn-signal.sh diff --git a/magnum/templates/swarm/fragments/configure-docker-registry.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure-docker-registry.sh similarity index 100% rename from magnum/templates/swarm/fragments/configure-docker-registry.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure-docker-registry.sh diff --git a/magnum/templates/swarm/fragments/configure-docker-storage.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure-docker-storage.sh similarity index 100% rename from magnum/templates/swarm/fragments/configure-docker-storage.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure-docker-storage.sh diff --git a/magnum/templates/swarm/fragments/configure-etcd.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure-etcd.sh similarity index 100% rename from magnum/templates/swarm/fragments/configure-etcd.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure-etcd.sh diff --git a/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure_docker_storage_driver_atomic.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure_docker_storage_driver_atomic.sh new file mode 100644 index 0000000000..26971a2608 --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/configure_docker_storage_driver_atomic.sh @@ -0,0 +1,38 @@ +# This file contains docker storage drivers configuration for fedora +# atomic hosts. Currently, devicemapper and overlay are supported. + +# Remove any existing docker-storage configuration. In case of an +# existing configuration, docker-storage-setup will fail. +clear_docker_storage_congiguration () { + if [ -f /etc/sysconfig/docker-storage ]; then + sed -i "/^DOCKER_STORAGE_OPTIONS=/ s/=.*/=/" /etc/sysconfig/docker-storage + fi +} + +# Configure docker storage with xfs as backing filesystem. +configure_overlay () { + clear_docker_storage_congiguration + + rm -rf /var/lib/docker/* + + mkfs.xfs ${device_path} + echo "${device_path} /var/lib/docker xfs defaults 0 0" >> /etc/fstab + mount -a + + echo "STORAGE_DRIVER=overlay" > /etc/sysconfig/docker-storage-setup + + # SELinux must be enabled and in enforcing mode on the physical + # machine, but must be disabled in the container when performing + # container separation + sed -i "/^OPTIONS=/ s/--selinux-enabled/--selinux-enabled=false/" /etc/sysconfig/docker +} + +# Configure docker storage with devicemapper using direct LVM +configure_devicemapper () { + clear_docker_storage_congiguration + + pvcreate ${device_path} + vgcreate docker ${device_path} + + echo "VG=docker" > /etc/sysconfig/docker-storage-setup +} diff --git a/magnum/templates/swarm/fragments/disable-selinux.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/disable-selinux.sh similarity index 100% rename from magnum/templates/swarm/fragments/disable-selinux.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/disable-selinux.sh diff --git a/magnum/templates/swarm/fragments/enable-docker-registry.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-docker-registry.sh similarity index 100% rename from magnum/templates/swarm/fragments/enable-docker-registry.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-docker-registry.sh diff --git a/magnum/templates/swarm/fragments/enable-services.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-services.sh similarity index 100% rename from magnum/templates/swarm/fragments/enable-services.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/enable-services.sh diff --git a/magnum/templates/swarm/fragments/make-cert.py b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/make-cert.py similarity index 100% rename from magnum/templates/swarm/fragments/make-cert.py rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/make-cert.py diff --git a/magnum/templates/swarm/fragments/network-config-service.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/network-config-service.sh similarity index 100% rename from magnum/templates/swarm/fragments/network-config-service.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/network-config-service.sh diff --git a/magnum/templates/swarm/fragments/network-service.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/network-service.sh similarity index 100% rename from magnum/templates/swarm/fragments/network-service.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/network-service.sh diff --git a/magnum/templates/swarm/fragments/remove-docker-key.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/remove-docker-key.sh similarity index 100% rename from magnum/templates/swarm/fragments/remove-docker-key.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/remove-docker-key.sh diff --git a/magnum/templates/swarm/fragments/write-bay-failure-service.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-bay-failure-service.yaml similarity index 100% rename from magnum/templates/swarm/fragments/write-bay-failure-service.yaml rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-bay-failure-service.yaml diff --git a/magnum/templates/swarm/fragments/write-docker-service.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-service.sh similarity index 100% rename from magnum/templates/swarm/fragments/write-docker-service.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-service.sh diff --git a/magnum/templates/swarm/fragments/write-docker-socket.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-socket.yaml similarity index 100% rename from magnum/templates/swarm/fragments/write-docker-socket.yaml rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-docker-socket.yaml diff --git a/magnum/templates/swarm/fragments/write-heat-params-master.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.yaml similarity index 100% rename from magnum/templates/swarm/fragments/write-heat-params-master.yaml rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-master.yaml diff --git a/magnum/templates/swarm/fragments/write-heat-params-node.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.yaml similarity index 100% rename from magnum/templates/swarm/fragments/write-heat-params-node.yaml rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-heat-params-node.yaml diff --git a/magnum/templates/swarm/fragments/write-network-config.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-network-config.sh similarity index 100% rename from magnum/templates/swarm/fragments/write-network-config.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-network-config.sh diff --git a/magnum/templates/swarm/fragments/write-swarm-agent-service.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-agent-service.sh similarity index 100% rename from magnum/templates/swarm/fragments/write-swarm-agent-service.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-agent-service.sh diff --git a/magnum/templates/swarm/fragments/write-swarm-master-service.sh b/magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-master-service.sh similarity index 100% rename from magnum/templates/swarm/fragments/write-swarm-master-service.sh rename to magnum/drivers/swarm_fedora_atomic_v1/templates/fragments/write-swarm-master-service.sh diff --git a/magnum/templates/swarm/swarmmaster.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml similarity index 99% rename from magnum/templates/swarm/swarmmaster.yaml rename to magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml index 29dc1cfe0c..8504859297 100644 --- a/magnum/templates/swarm/swarmmaster.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmmaster.yaml @@ -237,7 +237,7 @@ resources: config: str_replace: params: - $configure_docker_storage_driver: {get_file: ../common/fragments/configure_docker_storage_driver_atomic.sh} + $configure_docker_storage_driver: {get_file: fragments/configure_docker_storage_driver_atomic.sh} template: {get_file: fragments/configure-docker-storage.sh} make_cert: @@ -423,4 +423,3 @@ outputs: value: {get_attr: [swarm_master_floating, floating_ip_address]} description: > This is the "public" ip addresses of Swarm master. - diff --git a/magnum/templates/swarm/swarmnode.yaml b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml similarity index 98% rename from magnum/templates/swarm/swarmnode.yaml rename to magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml index 40e0750112..a9260fe67f 100644 --- a/magnum/templates/swarm/swarmnode.yaml +++ b/magnum/drivers/swarm_fedora_atomic_v1/templates/swarmnode.yaml @@ -237,7 +237,7 @@ resources: config: str_replace: params: - $configure_docker_storage_driver: {get_file: ../common/fragments/configure_docker_storage_driver_atomic.sh} + $configure_docker_storage_driver: {get_file: fragments/configure_docker_storage_driver_atomic.sh} template: {get_file: fragments/configure-docker-storage.sh} configure_docker_registry: diff --git a/magnum/drivers/swarm_fedora_atomic_v1/version.py b/magnum/drivers/swarm_fedora_atomic_v1/version.py new file mode 100644 index 0000000000..7889a173d6 --- /dev/null +++ b/magnum/drivers/swarm_fedora_atomic_v1/version.py @@ -0,0 +1,16 @@ +# Copyright 2016 - Rackspace Hosting +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version = '1.0.0' +driver = 'swarm_atomic' diff --git a/magnum/tests/functional/k8s/test_templates.py b/magnum/tests/functional/k8s/test_templates.py index cd66048811..fd4f5b017a 100644 --- a/magnum/tests/functional/k8s/test_templates.py +++ b/magnum/tests/functional/k8s/test_templates.py @@ -11,7 +11,7 @@ # under the License. -from magnum.conductor import template_definition as tdef +from magnum.drivers.common import template_def as tdef from magnum.tests import base diff --git a/magnum/tests/unit/conductor/test_template_definition.py b/magnum/tests/unit/conductor/test_template_definition.py index c931bce27f..46172582d8 100644 --- a/magnum/tests/unit/conductor/test_template_definition.py +++ b/magnum/tests/unit/conductor/test_template_definition.py @@ -14,22 +14,25 @@ import mock from oslo_config import cfg -from requests import exceptions as req_exceptions from magnum.common import exception from magnum.conductor import template_definition as tdef +from magnum.drivers.common import template_def as cmn_tdef +from magnum.drivers.swarm_fedora_atomic_v1 import template_def as swarm_tdef from magnum.tests import base +from requests import exceptions as req_exceptions + class TemplateDefinitionTestCase(base.TestCase): - @mock.patch.object(tdef, 'iter_entry_points') + @mock.patch.object(cmn_tdef, 'iter_entry_points') def test_load_entry_points(self, mock_iter_entry_points): mock_entry_point = mock.MagicMock() mock_entry_points = [mock_entry_point] mock_iter_entry_points.return_value = mock_entry_points.__iter__() - entry_points = tdef.TemplateDefinition.load_entry_points() + entry_points = cmn_tdef.TemplateDefinition.load_entry_points() for (expected_entry_point, (actual_entry_point, loaded_cls)) in zip(mock_entry_points, @@ -38,7 +41,7 @@ class TemplateDefinitionTestCase(base.TestCase): expected_entry_point.load.assert_called_once_with(require=False) def test_get_template_definitions(self): - defs = tdef.TemplateDefinition.get_template_definitions() + defs = cmn_tdef.TemplateDefinition.get_template_definitions() vm_atomic_k8s = defs[('vm', 'fedora-atomic', 'kubernetes')] vm_coreos_k8s = defs[('vm', 'coreos', 'kubernetes')] @@ -51,7 +54,7 @@ class TemplateDefinitionTestCase(base.TestCase): vm_coreos_k8s['magnum_vm_coreos_k8s']) def test_get_vm_atomic_kubernetes_definition(self): - definition = tdef.TemplateDefinition.get_template_definition( + definition = cmn_tdef.TemplateDefinition.get_template_definition( 'vm', 'fedora-atomic', 'kubernetes') @@ -60,7 +63,7 @@ class TemplateDefinitionTestCase(base.TestCase): tdef.AtomicK8sTemplateDefinition) def test_get_vm_coreos_kubernetes_definition(self): - definition = tdef.TemplateDefinition.get_template_definition( + definition = cmn_tdef.TemplateDefinition.get_template_definition( 'vm', 'coreos', 'kubernetes') @@ -69,16 +72,16 @@ class TemplateDefinitionTestCase(base.TestCase): tdef.CoreOSK8sTemplateDefinition) def test_get_vm_atomic_swarm_definition(self): - definition = tdef.TemplateDefinition.get_template_definition( + definition = cmn_tdef.TemplateDefinition.get_template_definition( 'vm', 'fedora-atomic', 'swarm') self.assertIsInstance(definition, - tdef.AtomicSwarmTemplateDefinition) + swarm_tdef.AtomicSwarmTemplateDefinition) def test_get_vm_ubuntu_mesos_definition(self): - definition = tdef.TemplateDefinition.get_template_definition( + definition = cmn_tdef.TemplateDefinition.get_template_definition( 'vm', 'ubuntu', 'mesos') @@ -88,7 +91,7 @@ class TemplateDefinitionTestCase(base.TestCase): def test_get_definition_not_supported(self): self.assertRaises(exception.BayTypeNotSupported, - tdef.TemplateDefinition.get_template_definition, + cmn_tdef.TemplateDefinition.get_template_definition, 'vm', 'not_supported', 'kubernetes') def test_get_definition_not_enabled(self): @@ -96,12 +99,12 @@ class TemplateDefinitionTestCase(base.TestCase): ['magnum_vm_atomic_k8s'], group='bay') self.assertRaises(exception.BayTypeNotEnabled, - tdef.TemplateDefinition.get_template_definition, + cmn_tdef.TemplateDefinition.get_template_definition, 'vm', 'coreos', 'kubernetes') def test_required_param_not_set(self): - param = tdef.ParameterMapping('test', baymodel_attr='test', - required=True) + param = cmn_tdef.ParameterMapping('test', baymodel_attr='test', + required=True) mock_baymodel = mock.MagicMock() mock_baymodel.test = None @@ -125,26 +128,26 @@ class TemplateDefinitionTestCase(base.TestCase): mock_stack = mock.MagicMock() mock_stack.to_dict.return_value = {'outputs': heat_outputs} - output = tdef.OutputMapping('key1') + output = cmn_tdef.OutputMapping('key1') value = output.get_output_value(mock_stack) self.assertEqual('value1', value) - output = tdef.OutputMapping('key2') + output = cmn_tdef.OutputMapping('key2') value = output.get_output_value(mock_stack) self.assertEqual(["value2", "value3"], value) - output = tdef.OutputMapping('key3') + output = cmn_tdef.OutputMapping('key3') value = output.get_output_value(mock_stack) self.assertIsNone(value) # verify stack with no 'outputs' attribute mock_stack.to_dict.return_value = {} - output = tdef.OutputMapping('key1') + output = cmn_tdef.OutputMapping('key1') value = output.get_output_value(mock_stack) self.assertIsNone(value) def test_add_output_with_mapping_type(self): - definition = tdef.TemplateDefinition.get_template_definition( + definition = cmn_tdef.TemplateDefinition.get_template_definition( 'vm', 'fedora-atomic', 'kubernetes') @@ -166,9 +169,9 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase): @mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.conductor.template_definition' '.AtomicK8sTemplateDefinition.get_discovery_url') - @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.BaseTemplateDefinition' '.get_params') - @mock.patch('magnum.conductor.template_definition.TemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.TemplateDefinition' '.get_output') def test_k8s_get_params(self, mock_get_output, mock_get_params, mock_get_discovery_url, mock_osc_class): @@ -219,9 +222,9 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase): @mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.conductor.template_definition' '.AtomicK8sTemplateDefinition.get_discovery_url') - @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.BaseTemplateDefinition' '.get_params') - @mock.patch('magnum.conductor.template_definition.TemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.TemplateDefinition' '.get_output') def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params, mock_get_discovery_url, mock_osc_class): @@ -371,7 +374,7 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase): def _test_update_outputs_api_address(self, coe, params, tls=True): - definition = tdef.TemplateDefinition.get_template_definition( + definition = cmn_tdef.TemplateDefinition.get_template_definition( 'vm', 'fedora-atomic', coe) @@ -438,7 +441,7 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase): def _test_update_outputs_none_api_address(self, coe, params, tls=True): - definition = tdef.TemplateDefinition.get_template_definition( + definition = cmn_tdef.TemplateDefinition.get_template_definition( 'vm', 'fedora-atomic', coe) @@ -483,11 +486,11 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase): class AtomicSwarmTemplateDefinitionTestCase(base.TestCase): @mock.patch('magnum.common.clients.OpenStackClients') - @mock.patch('magnum.conductor.template_definition' + @mock.patch('magnum.drivers.swarm_fedora_atomic_v1.template_def' '.AtomicSwarmTemplateDefinition.get_discovery_url') - @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.BaseTemplateDefinition' '.get_params') - @mock.patch('magnum.conductor.template_definition.TemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.TemplateDefinition' '.get_output') def test_swarm_get_params(self, mock_get_output, mock_get_params, mock_get_discovery_url, mock_osc_class): @@ -513,7 +516,7 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase): flannel_subnet = mock_baymodel.labels.get('flannel_network_subnetlen') flannel_backend = mock_baymodel.labels.get('flannel_backend') - swarm_def = tdef.AtomicSwarmTemplateDefinition() + swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition() swarm_def.get_params(mock_context, mock_baymodel, mock_bay) @@ -582,7 +585,7 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase): mock_bay = mock.MagicMock() mock_bay.discovery_url = None - swarm_def = tdef.AtomicSwarmTemplateDefinition() + swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition() discovery_url = swarm_def.get_discovery_url(mock_bay) mock_get.assert_called_once_with('http://etcd/test?size=1') @@ -603,13 +606,13 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase): fake_bay) def test_swarm_get_heat_param(self): - swarm_def = tdef.AtomicSwarmTemplateDefinition() + swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition() heat_param = swarm_def.get_heat_param(bay_attr='node_count') self.assertEqual('number_of_nodes', heat_param) def test_update_outputs(self): - swarm_def = tdef.AtomicSwarmTemplateDefinition() + swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition() expected_api_address = 'updated_address' expected_node_addresses = ['ex_minion', 'address'] @@ -645,9 +648,9 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase): class UbuntuMesosTemplateDefinitionTestCase(base.TestCase): @mock.patch('magnum.common.clients.OpenStackClients') - @mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.BaseTemplateDefinition' '.get_params') - @mock.patch('magnum.conductor.template_definition.TemplateDefinition' + @mock.patch('magnum.drivers.common.template_def.TemplateDefinition' '.get_output') def test_mesos_get_params(self, mock_get_output, mock_get_params, mock_osc_class): diff --git a/setup.cfg b/setup.cfg index dfe62d5520..c84b370d76 100644 --- a/setup.cfg +++ b/setup.cfg @@ -58,7 +58,7 @@ oslo.config.opts.defaults = magnum.template_definitions = magnum_vm_atomic_k8s = magnum.conductor.template_definition:AtomicK8sTemplateDefinition magnum_vm_coreos_k8s = magnum.conductor.template_definition:CoreOSK8sTemplateDefinition - magnum_vm_atomic_swarm = magnum.conductor.template_definition:AtomicSwarmTemplateDefinition + magnum_vm_atomic_swarm = magnum.drivers.swarm_fedora_atomic_v1.template_def:AtomicSwarmTemplateDefinition magnum_vm_ubuntu_mesos = magnum.conductor.template_definition:UbuntuMesosTemplateDefinition magnum.database.migration_backend =