From 64f0310042cd530063e877dea9f1bbe26e171fdf Mon Sep 17 00:00:00 2001 From: marios Date: Wed, 16 Oct 2013 12:31:39 +0300 Subject: [PATCH] Use TripleO Heat Merge to manage the stack Generate the YAML file describing Tuskar compute nodes and invoke Heat Merge. This review adds the master branch of tripleo-heat-templates as a requirement. At the moment this patch hardcodes the parameters which define the different Overcloud Roles. On a create over the Overcloud it creates one controller and one compute Overcloud role. When the Overcloud is updated it created a second compute node. This can be manually tested with these commands. Create the overcloud: curl -H "Content-Type:application/json"\ -XPOST http://localhost:8585/v1/overclouds/ -d '{}' Update the overcloud: curl -H "Content-Type:application/json"\ -XPUT http://localhost:8585/v1/overclouds/1 -d '{}' Delete the overcloud: curl -H "Content-Type:application/json"\ -XDELETE http://localhost:8585/v1/overclouds/1 -d '{}' Change-Id: I578b4e9f238590ea245b827bc75d252568d194fe --- requirements.txt | 2 + setup.cfg | 2 +- tuskar/api/controllers/v1/overcloud.py | 85 +++++++++- tuskar/common/exception.py | 20 +++ tuskar/heat/client.py | 34 ++-- tuskar/heat/template_tools.py | 78 +++++++++ .../api/controllers/v1/test_overcloud.py | 158 +++++++++++++++++- tuskar/tests/conf_fixture.py | 2 + tuskar/tests/heat/__init__.py | 0 tuskar/tests/heat/test_template_tools.py | 53 ++++++ 10 files changed, 413 insertions(+), 21 deletions(-) create mode 100644 tuskar/heat/template_tools.py create mode 100644 tuskar/tests/heat/__init__.py create mode 100644 tuskar/tests/heat/test_template_tools.py diff --git a/requirements.txt b/requirements.txt index 291a5a1e..e0b08333 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,5 @@ WSME>=0.5b6 python-novaclient>=2.15.0 PyYAML>=3.1.0 python-heatclient>=0.2.3 + +-e git+http://git.openstack.org/cgit/openstack/tripleo-heat-templates#egg=tripleo_heat_templates-master diff --git a/setup.cfg b/setup.cfg index 2a69ab5d..a83f3573 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,7 +34,7 @@ build-dir = doc/build source-dir = doc/source [egg_info] -tag_build = +tag_build = tag_date = 0 tag_svn_revision = 0 diff --git a/tuskar/api/controllers/v1/overcloud.py b/tuskar/api/controllers/v1/overcloud.py index 2f233de0..8b7317fd 100644 --- a/tuskar/api/controllers/v1/overcloud.py +++ b/tuskar/api/controllers/v1/overcloud.py @@ -1,5 +1,3 @@ -# -*- encoding: utf-8 -*- -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -14,19 +12,70 @@ import logging import pecan -from pecan import rest import wsme + +from pecan import rest from wsmeext import pecan as wsme_pecan from tuskar.api.controllers.v1 import models - +from tuskar.common import exception +from tuskar.heat.client import HeatClient +import tuskar.heat.template_tools as template_tools LOG = logging.getLogger(__name__) +# FIXME(lsmola) mocked params for POC, remove later by real ones +POC_PARAMS = {'controller': 1, 'compute': 1} +POC_PARAMS_UPDATE = {'controller': 1, 'compute': 2} + + +def process_stack(params, create=False): + """Helper function for processing the stack. Given a params dict containing + the Overcloud Roles and initialization parameters create or update the + stack. + + :param params: Dictionary of initialization params and overcloud roles for + heat template and initialization of stack/ + :type params: dict + + :param create: A flag to designate if we are creating or updating the stack + :type create: bool + """ + + overcloud = template_tools.merge_templates(params) + heat_client = HeatClient() + + stack_exists = heat_client.exists_stack() + if not heat_client.validate_template(overcloud): + raise exception.InvalidHeatTemplate() + + if stack_exists and create: + raise exception.StackAlreadyCreated() + + elif not stack_exists and not create: + raise exception.StackNotFound() + + res = heat_client.create_stack(overcloud, params) + + if not res: + if create: + raise exception.HeatTemplateCreateFailed() + + raise exception.HeatTemplateUpdateFailed() + + class OvercloudsController(rest.RestController): """REST controller for the Overcloud class.""" + _custom_actions = {'template_get': ['GET']} + + # FIXME(lsmola) this is for debugging purposes only, remove before I3 + @pecan.expose() + def template_get(self): + overcloud = template_tools.merge_templates(POC_PARAMS) + return overcloud + @wsme.validate(models.Overcloud) @wsme_pecan.wsexpose(models.Overcloud, body=models.Overcloud, @@ -55,6 +104,14 @@ class OvercloudsController(rest.RestController): saved_overcloud =\ models.Overcloud.from_db_model(result) + # FIXME(lsmola) This is just POC of creating a stack + # this has to be done properly with proper Work-flow abstraction of: + # step one- build template and start stack-create + # step 2- put the right stack_id to the overcloud + # step 3- initialize the stack + # step 4- set the correct overcloud status + process_stack(POC_PARAMS, create=True) + return saved_overcloud @wsme.validate(models.Overcloud) @@ -90,6 +147,12 @@ class OvercloudsController(rest.RestController): updated = models.Overcloud.from_db_model(result) + # FIXME(lsmola) This is just POC of updating a stack + # this probably should also have workflow + # step one- build template and stack-update + # step 2- set the correct overcloud status + process_stack(POC_PARAMS_UPDATE) + return updated @wsme_pecan.wsexpose(None, int, status_code=204) @@ -103,9 +166,23 @@ class OvercloudsController(rest.RestController): is no overcloud with the given ID """ + # FIXME(lsmola) this should always try to delete both overcloud + # and stack. So it requires some exception catch over below. LOG.debug('Deleting overcloud with ID: %s' % overcloud_id) pecan.request.dbapi.delete_overcloud_by_id(overcloud_id) + heat_client = HeatClient() + if not heat_client.exists_stack(): + # If the stack doesn't exist, we have nothing else to do here. + return + + result = heat_client.delete_stack() + + if not result: + raise wsme.exc.ClientSideError(_( + "Failed to delete the Heat overcloud." + )) + @wsme_pecan.wsexpose(models.Overcloud, int) def get_one(self, overcloud_id): """Returns a specific overcloud. diff --git a/tuskar/common/exception.py b/tuskar/common/exception.py index ecd73ea2..bc9f92d3 100644 --- a/tuskar/common/exception.py +++ b/tuskar/common/exception.py @@ -170,3 +170,23 @@ class DuplicateAttribute(DuplicateEntry): class ConfigNotFound(TuskarException): message = _("Could not find config at %(path)s") + + +class InvalidHeatTemplate(TuskarException): + message = _("Validation of the Heat Template failed.") + + +class StackNotFound(NotFound): + message = _("The Stack for this Overcloud can't be found.") + + +class StackAlreadyCreated(DuplicateEntry): + message = _("The Stack for this Overcloud already exists.") + + +class HeatTemplateUpdateFailed(TuskarException): + message = _("The Heat template failed to update.") + + +class HeatTemplateCreateFailed(TuskarException): + message = _("The Heat template failed to create.") diff --git a/tuskar/heat/client.py b/tuskar/heat/client.py index 908e6e96..77e0b172 100644 --- a/tuskar/heat/client.py +++ b/tuskar/heat/client.py @@ -80,15 +80,16 @@ class HeatClient(object): try: keystone = ksclient.Client(**CONF.heat_keystone) endpoint = keystone.service_catalog.url_for( - service_type=CONF.heat['service_type'], - endpoint_type=CONF.heat['endpoint_type']) + service_type=CONF.heat['service_type'], + endpoint_type=CONF.heat['endpoint_type'] + ) self.connection = heatclient( endpoint=endpoint, token=keystone.auth_token, username=CONF.heat_keystone['username'], password=CONF.heat_keystone['password']) - except Exception as e: - LOG.exception(e) + except Exception: + LOG.exception("An error occurred initialising the HeatClient") self.connection = None def validate_template(self, template_body): @@ -96,8 +97,8 @@ class HeatClient(object): try: self.connection.stacks.validate(template=template_body) return True - except Exception as e: - LOG.exception(e) + except Exception: + LOG.exception("Validation of the Heat template failed.") return False def get_stack(self, name=None): @@ -110,8 +111,8 @@ class HeatClient(object): def get_template(self): """Get JSON representation of the Heat overcloud template.""" return self.connection.stacks.template( - stack_id=CONF.heat['stack_name'] - ) + stack_id=CONF.heat['stack_name'] + ) def update_stack(self, template_body, params): """Update the Heat overcloud stack.""" @@ -120,8 +121,17 @@ class HeatClient(object): template=template_body, parameters=params) return True - except Exception as e: - LOG.exception(e) + except Exception: + LOG.exception("An error occurred updating the stack.") + return False + + def delete_stack(self): + """Delete the Heat overcloud stack.""" + try: + self.connection.stacks.delete(stack_id=CONF.heat['stack_name']) + return True + except Exception: + LOG.exception("An error occurred deleting the stack.") return False def create_stack(self, template_body, params): @@ -131,8 +141,8 @@ class HeatClient(object): template=template_body, parameters=params) return True - except Exception as e: - LOG.exception(e) + except Exception: + LOG.exception("An error occurred creating the stack.") return False def exists_stack(self, name=None): diff --git a/tuskar/heat/template_tools.py b/tuskar/heat/template_tools.py new file mode 100644 index 00000000..b8ef3bbc --- /dev/null +++ b/tuskar/heat/template_tools.py @@ -0,0 +1,78 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Utilities for using merge.py to generate overcloud.yaml to hand over to Heat. +Translates Tuskar resources into the overcloud heat template, using merge.py +from upstream tripleo-heat-templates. +""" + +import os + +from oslo.config import cfg +from tripleo_heat_merge import merge + + +# The name of the compute Overcloud role - defined for special case handling +OVERCLOUD_COMPUTE_ROLE = 'compute' + + +def generate_scaling_params(overcloud_roles): + """Given a dictionary containing a key value mapping of Overcloud Role name + to a count of the nodes return the scaling parameters to be used by + tripleo_heat_merge + + :param overcloud_roles: Dictionary with role names and a count of the nodes + :type overcloud_roles: dict + + :return: scaling parameters dict + :rtype: dict + """ + + scaling = {} + + for overcloud_role, count in overcloud_roles.items(): + overcloud_role = overcloud_role.lower() + + if overcloud_role == OVERCLOUD_COMPUTE_ROLE: + scaling = dict(scaling.items() + + merge.parse_scaling(["NovaCompute=%s" % (count)]).items()) + + return scaling + + +def _join_template_path(file_name): + return os.path.abspath( + os.path.join(os.path.dirname(cfg.CONF.tht_local_dir), file_name) + ) + + +def merge_templates(overcloud_roles): + """Merge the Overcloud Roles with overcloud.yaml using merge from + tripleo_heat_merge + + See tripleo-heat-templates for further details. + """ + + # TODO(dmatthews): Add exception handling to catch merge errors + + scale_params = generate_scaling_params(overcloud_roles) + overcloud_src_path = _join_template_path("overcloud-source.yaml") + ssl_src_path = _join_template_path("ssl-source.yaml") + swift_src_path = _join_template_path("swift-source.yaml") + + template = merge.merge( + [overcloud_src_path, ssl_src_path, swift_src_path], None, None, + included_template_dir=cfg.CONF.tht_local_dir, scaling=scale_params + ) + + return template diff --git a/tuskar/tests/api/controllers/v1/test_overcloud.py b/tuskar/tests/api/controllers/v1/test_overcloud.py index c8e6c6d0..94e8e6ba 100644 --- a/tuskar/tests/api/controllers/v1/test_overcloud.py +++ b/tuskar/tests/api/controllers/v1/test_overcloud.py @@ -12,11 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. +import mock import os -import mock from pecan.testing import load_test_app +from tuskar.api.controllers.v1 import overcloud +from tuskar.common import exception from tuskar.db.sqlalchemy import models as db_models from tuskar.tests import base @@ -69,13 +71,85 @@ class OvercloudTests(base.TestCase): mock_db_get.assert_called_once_with(12345) + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': True, + 'exists_stack.return_value': False, + 'create_stack.return_value': True, + }) + ) + def test_create_stack(self, mock_heat_client, mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test + response = overcloud.process_stack({}, create=True) + + # Verify + self.assertEqual(response, None) + + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': True, + 'exists_stack.return_value': False, + 'create_stack.return_value': False, + }) + ) + def test_create_stack_heat_exception(self, mock_heat_client, + mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test and Verify + self.assertRaises( + exception.HeatTemplateCreateFailed, + overcloud.process_stack, {}, True) + + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': True, + 'exists_stack.return_value': True, + 'create_stack.return_value': True, + }) + ) + def test_create_stack_existing_exception(self, mock_heat_client, + mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test and Verify + self.assertRaises( + exception.StackAlreadyCreated, overcloud.process_stack, {}, True) + + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': False, + 'exists_stack.return_value': False, + 'create_stack.return_value': True, + }) + ) + def test_create_stack_not_valid_exception(self, mock_heat_client, + mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test and Verify + self.assertRaises( + exception.InvalidHeatTemplate, overcloud.process_stack, {}, True) + + @mock.patch('tuskar.api.controllers.v1.overcloud.process_stack') @mock.patch('tuskar.db.sqlalchemy.api.Connection.create_overcloud') - def test_post(self, mock_db_create): + def test_post(self, mock_db_create, mock_process_stack): # Setup create_me = {'name': 'new'} fake_created = db_models.Overcloud(name='created') mock_db_create.return_value = fake_created + mock_process_stack.return_value = None # Test response = self.app.post_json(URL_OVERCLOUDS, params=create_me) @@ -91,8 +165,78 @@ class OvercloudTests(base.TestCase): db_models.Overcloud)) self.assertEqual(db_create_model.name, create_me['name']) + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': True, + 'exists_stack.return_value': True, + 'create_stack.return_value': True, + }) + ) + def test_update_stack(self, mock_heat_client, mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test + response = overcloud.process_stack({}) + + # Verify + self.assertEqual(response, None) + + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': True, + 'exists_stack.return_value': True, + 'create_stack.return_value': False, + }) + ) + def test_update_stack_heat_exception(self, mock_heat_client, + mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test and Verify + self.assertRaises( + exception.HeatTemplateUpdateFailed, overcloud.process_stack, {}) + + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': True, + 'exists_stack.return_value': False, + 'create_stack.return_value': True, + }) + ) + def test_update_stack_not_existing_exception(self, mock_heat_client, + mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test and Verify + self.assertRaises( + exception.StackNotFound, overcloud.process_stack, {}) + + @mock.patch('tuskar.heat.template_tools.merge_templates') + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'validate_template.return_value': False, + 'exists_stack.return_value': True, + 'create_stack.return_value': True, + }) + ) + def test_update_stack_not_valid_exception(self, mock_heat_client, + mock_heat_merge_templates): + # Setup + mock_heat_merge_templates.return_value = None + + # Test and Verify + self.assertRaises( + exception.InvalidHeatTemplate, overcloud.process_stack, {}) + + @mock.patch('tuskar.api.controllers.v1.overcloud.process_stack') @mock.patch('tuskar.db.sqlalchemy.api.Connection.update_overcloud') - def test_put(self, mock_db_update): + def test_put(self, mock_db_update, mock_process_stack): # Setup changes = {'name': 'updated'} @@ -100,6 +244,7 @@ class OvercloudTests(base.TestCase): attributes=[], counts=[]) mock_db_update.return_value = fake_updated + mock_process_stack.return_value = None # Test url = URL_OVERCLOUDS + '/' + '12345' @@ -119,7 +264,12 @@ class OvercloudTests(base.TestCase): @mock.patch('tuskar.db.sqlalchemy.api.' 'Connection.delete_overcloud_by_id') - def test_delete(self, mock_db_delete): + @mock.patch( + 'tuskar.heat.client.HeatClient.__new__', return_value=mock.Mock(**{ + 'delete_stack.return_value': True, + }) + ) + def test_delete(self, mock_heat_client, mock_db_delete): # Test url = URL_OVERCLOUDS + '/' + '12345' response = self.app.delete(url) diff --git a/tuskar/tests/conf_fixture.py b/tuskar/tests/conf_fixture.py index 9eef83df..4839fd24 100644 --- a/tuskar/tests/conf_fixture.py +++ b/tuskar/tests/conf_fixture.py @@ -42,5 +42,7 @@ class ConfFixture(fixtures.Fixture): self.conf.set_default('sqlite_synchronous', False) self.conf.set_default('use_ipv6', True) self.conf.set_default('verbose', True) + self.conf.set_default('tht_local_dir', + '/etc/tuskar/tripleo-heat-templates/') config.parse_args([], default_config_files=[]) self.addCleanup(self.conf.reset) diff --git a/tuskar/tests/heat/__init__.py b/tuskar/tests/heat/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tuskar/tests/heat/test_template_tools.py b/tuskar/tests/heat/test_template_tools.py new file mode 100644 index 00000000..fcee983f --- /dev/null +++ b/tuskar/tests/heat/test_template_tools.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock +import unittest + +from tuskar.heat import template_tools + + +class TemplateToolsTests(unittest.TestCase): + + @mock.patch('tripleo_heat_merge.merge.parse_scaling') + def test_generate_scaling_params(self, mock_parse_scaling): + # Setup + overcloud_roles = {'controller': 1, 'compute': 12} + + # Test + template_tools.generate_scaling_params(overcloud_roles) + + # Verify + mock_parse_scaling.assert_called_once_with(['NovaCompute=12']) + + @mock.patch('tripleo_heat_merge.merge.merge') + def test_merge_templates(self, mock_merge): + # Setup + overcloud_roles = {'controller': 1, 'compute': 12} + + # Test + template_tools.merge_templates(overcloud_roles) + + # Verify + mock_merge.assert_called_once_with([ + '/etc/tuskar/tripleo-heat-templates/overcloud-source.yaml', + '/etc/tuskar/tripleo-heat-templates/ssl-source.yaml', + '/etc/tuskar/tripleo-heat-templates/swift-source.yaml'], + None, + None, + scaling={ + 'NovaCompute0': 12 + }, + included_template_dir='/etc/tuskar/tripleo-heat-templates/' + )