From 242cbe00daf495a9e224f12e963f301e914dd278 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Wed, 6 Jul 2016 11:01:14 +0200 Subject: [PATCH] Mesos-Ubuntu bay driver implementation Moves templates and template_definitions to the new directory structure. Change-Id: I42e4d2bd056f3d8082ef51ef599d917f2fe82960 Paritially-Implements: blueprint bay-drivers --- magnum/conductor/template_definition.py | 75 ---------------- magnum/drivers/common/template_def.py | 5 -- .../mesos => drivers/mesos_ubuntu_v1}/COPYING | 0 .../mesos_ubuntu_v1}/README.md | 0 magnum/drivers/mesos_ubuntu_v1/__init__.py | 0 .../mesos_ubuntu_v1/image}/Dockerfile | 0 .../mesos_ubuntu_v1/image}/README.md | 0 .../image}/docker/elements-deps | 0 .../image}/docker/package-installs.yaml | 0 .../post-install.d/60-disable-docker-service | 0 .../docker/pre-install.d/10-add-docker-repo | 0 .../image}/mesos/elements-deps | 0 .../image}/mesos/package-installs.yaml | 0 .../mesos/post-install.d/60-disable-upstart | 0 .../image}/mesos/pre-install.d/10-apt-repo | 0 .../drivers/mesos_ubuntu_v1/template_def.py | 87 +++++++++++++++++++ .../templates}/fragments/add-proxy.sh | 0 .../fragments/configure-mesos-master.sh | 0 .../fragments/configure-mesos-slave.sh | 0 .../fragments/start-services-master.sh | 0 .../fragments/start-services-slave.sh | 0 .../templates}/fragments/volume-service.sh | 0 .../fragments/write-heat-params-master.sh | 0 .../fragments/write-heat-params.yaml | 0 .../templates}/mesoscluster.yaml | 0 .../templates}/mesosmaster.yaml | 0 .../templates}/mesosslave.yaml | 0 .../conductor/test_template_definition.py | 9 +- setup.cfg | 2 +- 29 files changed, 93 insertions(+), 85 deletions(-) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1}/COPYING (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1}/README.md (100%) create mode 100644 magnum/drivers/mesos_ubuntu_v1/__init__.py rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/image}/Dockerfile (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/README.md (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/docker/elements-deps (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/docker/package-installs.yaml (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/docker/post-install.d/60-disable-docker-service (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/docker/pre-install.d/10-add-docker-repo (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/mesos/elements-deps (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/mesos/package-installs.yaml (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/mesos/post-install.d/60-disable-upstart (100%) rename magnum/{templates/mesos/elements => drivers/mesos_ubuntu_v1/image}/mesos/pre-install.d/10-apt-repo (100%) create mode 100644 magnum/drivers/mesos_ubuntu_v1/template_def.py rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/add-proxy.sh (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/configure-mesos-master.sh (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/configure-mesos-slave.sh (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/start-services-master.sh (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/start-services-slave.sh (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/volume-service.sh (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/write-heat-params-master.sh (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/fragments/write-heat-params.yaml (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/mesoscluster.yaml (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/mesosmaster.yaml (100%) rename magnum/{templates/mesos => drivers/mesos_ubuntu_v1/templates}/mesosslave.yaml (100%) diff --git a/magnum/conductor/template_definition.py b/magnum/conductor/template_definition.py index 530783572e..2c637fe358 100644 --- a/magnum/conductor/template_definition.py +++ b/magnum/conductor/template_definition.py @@ -40,11 +40,6 @@ template_def_opts = [ cfg.StrOpt('etcd_discovery_service_endpoint_format', default='https://discovery.etcd.io/new?size=%(size)d', help=_('Url for etcd public discovery endpoint.')), - cfg.StrOpt('mesos_ubuntu_template_path', - default=paths.basedir_def('templates/mesos/' - 'mesoscluster.yaml'), - help=_('Location of template to build a Mesos cluster ' - 'on Ubuntu.')), cfg.ListOpt('enabled_definitions', default=['magnum_vm_atomic_k8s', 'magnum_vm_coreos_k8s', 'magnum_vm_atomic_swarm', 'magnum_vm_ubuntu_mesos'], @@ -206,73 +201,3 @@ class CoreOSK8sTemplateDefinition(K8sTemplateDefinition): @property def template_path(self): return cfg.CONF.bay.k8s_coreos_template_path - - -class UbuntuMesosTemplateDefinition(template_def.BaseTemplateDefinition): - """Mesos template for Ubuntu VM.""" - - provides = [ - {'server_type': 'vm', 'os': 'ubuntu', 'coe': 'mesos'}, - ] - - def __init__(self): - super(UbuntuMesosTemplateDefinition, self).__init__() - self.add_parameter('external_network', - baymodel_attr='external_network_id', - required=True) - self.add_parameter('number_of_slaves', - bay_attr='node_count') - self.add_parameter('master_flavor', - baymodel_attr='master_flavor_id') - self.add_parameter('slave_flavor', - baymodel_attr='flavor_id') - self.add_parameter('cluster_name', - bay_attr='name') - self.add_parameter('volume_driver', - baymodel_attr='volume_driver') - - self.add_output('api_address', - bay_attr='api_address') - self.add_output('mesos_master_private', - bay_attr=None) - self.add_output('mesos_master', - bay_attr='master_addresses') - self.add_output('mesos_slaves_private', - bay_attr=None) - self.add_output('mesos_slaves', - bay_attr='node_addresses') - - def get_params(self, context, baymodel, bay, **kwargs): - extra_params = kwargs.pop('extra_params', {}) - # HACK(apmelton) - This uses the user's bearer token, ideally - # it should be replaced with an actual trust token with only - # access to do what the template needs it to do. - osc = self.get_osc(context) - extra_params['auth_url'] = context.auth_url - extra_params['username'] = context.user_name - extra_params['tenant_name'] = context.tenant - extra_params['domain_name'] = context.domain_name - extra_params['region_name'] = osc.cinder_region_name() - - label_list = ['rexray_preempt', 'mesos_slave_isolation', - 'mesos_slave_image_providers', - 'mesos_slave_work_dir', - 'mesos_slave_executor_env_variables'] - - for label in label_list: - extra_params[label] = baymodel.labels.get(label) - - scale_mgr = kwargs.pop('scale_manager', None) - if scale_mgr: - hosts = self.get_output('mesos_slaves_private') - extra_params['slaves_to_remove'] = ( - scale_mgr.get_removal_nodes(hosts)) - - return super(UbuntuMesosTemplateDefinition, - self).get_params(context, baymodel, bay, - extra_params=extra_params, - **kwargs) - - @property - def template_path(self): - return cfg.CONF.bay.mesos_ubuntu_template_path diff --git a/magnum/drivers/common/template_def.py b/magnum/drivers/common/template_def.py index 0666a9eb86..b40fe02262 100644 --- a/magnum/drivers/common/template_def.py +++ b/magnum/drivers/common/template_def.py @@ -48,11 +48,6 @@ template_def_opts = [ cfg.StrOpt('etcd_discovery_service_endpoint_format', default='https://discovery.etcd.io/new?size=%(size)d', help=_('Url for etcd public discovery endpoint.')), - cfg.StrOpt('mesos_ubuntu_template_path', - default=paths.basedir_def('templates/mesos/' - 'mesoscluster.yaml'), - help=_('Location of template to build a Mesos cluster ' - 'on Ubuntu.')), cfg.ListOpt('enabled_definitions', default=['magnum_vm_atomic_k8s', 'magnum_vm_coreos_k8s', 'magnum_vm_atomic_swarm', 'magnum_vm_ubuntu_mesos'], diff --git a/magnum/templates/mesos/COPYING b/magnum/drivers/mesos_ubuntu_v1/COPYING similarity index 100% rename from magnum/templates/mesos/COPYING rename to magnum/drivers/mesos_ubuntu_v1/COPYING diff --git a/magnum/templates/mesos/README.md b/magnum/drivers/mesos_ubuntu_v1/README.md similarity index 100% rename from magnum/templates/mesos/README.md rename to magnum/drivers/mesos_ubuntu_v1/README.md diff --git a/magnum/drivers/mesos_ubuntu_v1/__init__.py b/magnum/drivers/mesos_ubuntu_v1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/magnum/templates/mesos/Dockerfile b/magnum/drivers/mesos_ubuntu_v1/image/Dockerfile similarity index 100% rename from magnum/templates/mesos/Dockerfile rename to magnum/drivers/mesos_ubuntu_v1/image/Dockerfile diff --git a/magnum/templates/mesos/elements/README.md b/magnum/drivers/mesos_ubuntu_v1/image/README.md similarity index 100% rename from magnum/templates/mesos/elements/README.md rename to magnum/drivers/mesos_ubuntu_v1/image/README.md diff --git a/magnum/templates/mesos/elements/docker/elements-deps b/magnum/drivers/mesos_ubuntu_v1/image/docker/elements-deps similarity index 100% rename from magnum/templates/mesos/elements/docker/elements-deps rename to magnum/drivers/mesos_ubuntu_v1/image/docker/elements-deps diff --git a/magnum/templates/mesos/elements/docker/package-installs.yaml b/magnum/drivers/mesos_ubuntu_v1/image/docker/package-installs.yaml similarity index 100% rename from magnum/templates/mesos/elements/docker/package-installs.yaml rename to magnum/drivers/mesos_ubuntu_v1/image/docker/package-installs.yaml diff --git a/magnum/templates/mesos/elements/docker/post-install.d/60-disable-docker-service b/magnum/drivers/mesos_ubuntu_v1/image/docker/post-install.d/60-disable-docker-service similarity index 100% rename from magnum/templates/mesos/elements/docker/post-install.d/60-disable-docker-service rename to magnum/drivers/mesos_ubuntu_v1/image/docker/post-install.d/60-disable-docker-service diff --git a/magnum/templates/mesos/elements/docker/pre-install.d/10-add-docker-repo b/magnum/drivers/mesos_ubuntu_v1/image/docker/pre-install.d/10-add-docker-repo similarity index 100% rename from magnum/templates/mesos/elements/docker/pre-install.d/10-add-docker-repo rename to magnum/drivers/mesos_ubuntu_v1/image/docker/pre-install.d/10-add-docker-repo diff --git a/magnum/templates/mesos/elements/mesos/elements-deps b/magnum/drivers/mesos_ubuntu_v1/image/mesos/elements-deps similarity index 100% rename from magnum/templates/mesos/elements/mesos/elements-deps rename to magnum/drivers/mesos_ubuntu_v1/image/mesos/elements-deps diff --git a/magnum/templates/mesos/elements/mesos/package-installs.yaml b/magnum/drivers/mesos_ubuntu_v1/image/mesos/package-installs.yaml similarity index 100% rename from magnum/templates/mesos/elements/mesos/package-installs.yaml rename to magnum/drivers/mesos_ubuntu_v1/image/mesos/package-installs.yaml diff --git a/magnum/templates/mesos/elements/mesos/post-install.d/60-disable-upstart b/magnum/drivers/mesos_ubuntu_v1/image/mesos/post-install.d/60-disable-upstart similarity index 100% rename from magnum/templates/mesos/elements/mesos/post-install.d/60-disable-upstart rename to magnum/drivers/mesos_ubuntu_v1/image/mesos/post-install.d/60-disable-upstart diff --git a/magnum/templates/mesos/elements/mesos/pre-install.d/10-apt-repo b/magnum/drivers/mesos_ubuntu_v1/image/mesos/pre-install.d/10-apt-repo similarity index 100% rename from magnum/templates/mesos/elements/mesos/pre-install.d/10-apt-repo rename to magnum/drivers/mesos_ubuntu_v1/image/mesos/pre-install.d/10-apt-repo diff --git a/magnum/drivers/mesos_ubuntu_v1/template_def.py b/magnum/drivers/mesos_ubuntu_v1/template_def.py new file mode 100644 index 0000000000..42c6c90865 --- /dev/null +++ b/magnum/drivers/mesos_ubuntu_v1/template_def.py @@ -0,0 +1,87 @@ +# Copyright 2016 Rackspace Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import os + +from magnum.drivers.common import template_def + + +class UbuntuMesosTemplateDefinition(template_def.BaseTemplateDefinition): + """Mesos template for Ubuntu VM.""" + + provides = [ + {'server_type': 'vm', 'os': 'ubuntu', 'coe': 'mesos'}, + ] + + def __init__(self): + super(UbuntuMesosTemplateDefinition, self).__init__() + self.add_parameter('external_network', + baymodel_attr='external_network_id', + required=True) + self.add_parameter('number_of_slaves', + bay_attr='node_count') + self.add_parameter('master_flavor', + baymodel_attr='master_flavor_id') + self.add_parameter('slave_flavor', + baymodel_attr='flavor_id') + self.add_parameter('cluster_name', + bay_attr='name') + self.add_parameter('volume_driver', + baymodel_attr='volume_driver') + + self.add_output('api_address', + bay_attr='api_address') + self.add_output('mesos_master_private', + bay_attr=None) + self.add_output('mesos_master', + bay_attr='master_addresses') + self.add_output('mesos_slaves_private', + bay_attr=None) + self.add_output('mesos_slaves', + bay_attr='node_addresses') + + def get_params(self, context, baymodel, bay, **kwargs): + extra_params = kwargs.pop('extra_params', {}) + # HACK(apmelton) - This uses the user's bearer token, ideally + # it should be replaced with an actual trust token with only + # access to do what the template needs it to do. + osc = self.get_osc(context) + extra_params['auth_url'] = context.auth_url + extra_params['username'] = context.user_name + extra_params['tenant_name'] = context.tenant + extra_params['domain_name'] = context.domain_name + extra_params['region_name'] = osc.cinder_region_name() + + label_list = ['rexray_preempt', 'mesos_slave_isolation', + 'mesos_slave_image_providers', + 'mesos_slave_work_dir', + 'mesos_slave_executor_env_variables'] + + for label in label_list: + extra_params[label] = baymodel.labels.get(label) + + scale_mgr = kwargs.pop('scale_manager', None) + if scale_mgr: + hosts = self.get_output('mesos_slaves_private') + extra_params['slaves_to_remove'] = ( + scale_mgr.get_removal_nodes(hosts)) + + return super(UbuntuMesosTemplateDefinition, + self).get_params(context, baymodel, bay, + extra_params=extra_params, + **kwargs) + + @property + def template_path(self): + return os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'templates/mesoscluster.yaml') diff --git a/magnum/templates/mesos/fragments/add-proxy.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/add-proxy.sh similarity index 100% rename from magnum/templates/mesos/fragments/add-proxy.sh rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/add-proxy.sh diff --git a/magnum/templates/mesos/fragments/configure-mesos-master.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-master.sh similarity index 100% rename from magnum/templates/mesos/fragments/configure-mesos-master.sh rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-master.sh diff --git a/magnum/templates/mesos/fragments/configure-mesos-slave.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-slave.sh similarity index 100% rename from magnum/templates/mesos/fragments/configure-mesos-slave.sh rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/configure-mesos-slave.sh diff --git a/magnum/templates/mesos/fragments/start-services-master.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/start-services-master.sh similarity index 100% rename from magnum/templates/mesos/fragments/start-services-master.sh rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/start-services-master.sh diff --git a/magnum/templates/mesos/fragments/start-services-slave.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/start-services-slave.sh similarity index 100% rename from magnum/templates/mesos/fragments/start-services-slave.sh rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/start-services-slave.sh diff --git a/magnum/templates/mesos/fragments/volume-service.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/volume-service.sh similarity index 100% rename from magnum/templates/mesos/fragments/volume-service.sh rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/volume-service.sh diff --git a/magnum/templates/mesos/fragments/write-heat-params-master.sh b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/write-heat-params-master.sh similarity index 100% rename from magnum/templates/mesos/fragments/write-heat-params-master.sh rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/write-heat-params-master.sh diff --git a/magnum/templates/mesos/fragments/write-heat-params.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/fragments/write-heat-params.yaml similarity index 100% rename from magnum/templates/mesos/fragments/write-heat-params.yaml rename to magnum/drivers/mesos_ubuntu_v1/templates/fragments/write-heat-params.yaml diff --git a/magnum/templates/mesos/mesoscluster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml similarity index 100% rename from magnum/templates/mesos/mesoscluster.yaml rename to magnum/drivers/mesos_ubuntu_v1/templates/mesoscluster.yaml diff --git a/magnum/templates/mesos/mesosmaster.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml similarity index 100% rename from magnum/templates/mesos/mesosmaster.yaml rename to magnum/drivers/mesos_ubuntu_v1/templates/mesosmaster.yaml diff --git a/magnum/templates/mesos/mesosslave.yaml b/magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml similarity index 100% rename from magnum/templates/mesos/mesosslave.yaml rename to magnum/drivers/mesos_ubuntu_v1/templates/mesosslave.yaml diff --git a/magnum/tests/unit/conductor/test_template_definition.py b/magnum/tests/unit/conductor/test_template_definition.py index 46172582d8..c91000fa18 100644 --- a/magnum/tests/unit/conductor/test_template_definition.py +++ b/magnum/tests/unit/conductor/test_template_definition.py @@ -18,6 +18,7 @@ from oslo_config import cfg from magnum.common import exception from magnum.conductor import template_definition as tdef from magnum.drivers.common import template_def as cmn_tdef +from magnum.drivers.mesos_ubuntu_v1 import template_def as mesos_tdef from magnum.drivers.swarm_fedora_atomic_v1 import template_def as swarm_tdef from magnum.tests import base @@ -87,7 +88,7 @@ class TemplateDefinitionTestCase(base.TestCase): 'mesos') self.assertIsInstance(definition, - tdef.UbuntuMesosTemplateDefinition) + mesos_tdef.UbuntuMesosTemplateDefinition) def test_get_definition_not_supported(self): self.assertRaises(exception.BayTypeNotSupported, @@ -680,7 +681,7 @@ class UbuntuMesosTemplateDefinitionTestCase(base.TestCase): mock_scale_manager = mock.MagicMock() mock_scale_manager.get_removal_nodes.return_value = removal_nodes - mesos_def = tdef.UbuntuMesosTemplateDefinition() + mesos_def = mesos_tdef.UbuntuMesosTemplateDefinition() mesos_def.get_params(mock_context, mock_baymodel, mock_bay, scale_manager=mock_scale_manager) @@ -702,7 +703,7 @@ class UbuntuMesosTemplateDefinitionTestCase(base.TestCase): mock_bay, **expected_kwargs) def test_mesos_get_heat_param(self): - mesos_def = tdef.UbuntuMesosTemplateDefinition() + mesos_def = mesos_tdef.UbuntuMesosTemplateDefinition() heat_param = mesos_def.get_heat_param(bay_attr='node_count') self.assertEqual('number_of_slaves', heat_param) @@ -711,7 +712,7 @@ class UbuntuMesosTemplateDefinitionTestCase(base.TestCase): self.assertEqual('number_of_masters', heat_param) def test_update_outputs(self): - mesos_def = tdef.UbuntuMesosTemplateDefinition() + mesos_def = mesos_tdef.UbuntuMesosTemplateDefinition() expected_api_address = 'updated_address' expected_node_addresses = ['ex_slave', 'address'] diff --git a/setup.cfg b/setup.cfg index c84b370d76..ee674a3d54 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,7 +59,7 @@ magnum.template_definitions = magnum_vm_atomic_k8s = magnum.conductor.template_definition:AtomicK8sTemplateDefinition magnum_vm_coreos_k8s = magnum.conductor.template_definition:CoreOSK8sTemplateDefinition magnum_vm_atomic_swarm = magnum.drivers.swarm_fedora_atomic_v1.template_def:AtomicSwarmTemplateDefinition - magnum_vm_ubuntu_mesos = magnum.conductor.template_definition:UbuntuMesosTemplateDefinition + magnum_vm_ubuntu_mesos = magnum.drivers.mesos_ubuntu_v1.template_def:UbuntuMesosTemplateDefinition magnum.database.migration_backend = sqlalchemy = magnum.db.sqlalchemy.migration