From 3ab4f15a43aee3d943f080b6fa7a67c8f25eee61 Mon Sep 17 00:00:00 2001 From: rabi Date: Thu, 8 Mar 2018 22:15:50 +0530 Subject: [PATCH] Option for retrieving child templates and env files from swift This provides an option to specify a swift container for stack actions and all child templates and env files will be fetched from the container, if available. However, files coming in the 'files' map from the client will have precedence, if the same is also present in swift. Change-Id: Ifa21fbcb41fcb77827997cce2d5e9266ba849b17 Story: #1755453 Task: 17353 --- heat/api/middleware/fault.py | 1 + heat/api/openstack/v1/stacks.py | 25 +++-- heat/common/exception.py | 4 + heat/engine/clients/os/swift.py | 38 ++++++++ heat/engine/service.py | 70 +++++++++----- heat/engine/template_files.py | 19 ++++ heat/rpc/client.py | 45 ++++++--- heat/tests/api/cfn/test_api_cfn_v1.py | 21 +++-- heat/tests/api/openstack_v1/test_stacks.py | 66 ++++++++----- .../engine/service/test_service_engine.py | 2 +- .../tests/engine/service/test_stack_create.py | 92 +++++++++++++++---- .../tests/engine/service/test_stack_update.py | 42 +++++++-- heat/tests/test_engine_service.py | 9 +- heat/tests/test_rpc_client.py | 5 + 14 files changed, 334 insertions(+), 105 deletions(-) diff --git a/heat/api/middleware/fault.py b/heat/api/middleware/fault.py index 2e977393a7..a190ff84d6 100644 --- a/heat/api/middleware/fault.py +++ b/heat/api/middleware/fault.py @@ -82,6 +82,7 @@ class FaultWrapper(wsgi.Middleware): 'MissingCredentialError': webob.exc.HTTPBadRequest, 'UserParameterMissing': webob.exc.HTTPBadRequest, 'RequestLimitExceeded': webob.exc.HTTPBadRequest, + 'DownloadLimitExceeded': webob.exc.HTTPBadRequest, 'Invalid': webob.exc.HTTPBadRequest, 'ResourcePropertyConflict': webob.exc.HTTPBadRequest, 'PropertyUnspecifiedError': webob.exc.HTTPBadRequest, diff --git a/heat/api/openstack/v1/stacks.py b/heat/api/openstack/v1/stacks.py index cbbbded381..d144ea1972 100644 --- a/heat/api/openstack/v1/stacks.py +++ b/heat/api/openstack/v1/stacks.py @@ -50,6 +50,7 @@ class InstantiationData(object): PARAM_ENVIRONMENT, PARAM_FILES, PARAM_ENVIRONMENT_FILES, + PARAM_FILES_CONTAINER ) = ( 'stack_name', 'template', @@ -58,6 +59,7 @@ class InstantiationData(object): 'environment', 'files', 'environment_files', + 'files_container' ) def __init__(self, data, patch=False): @@ -157,6 +159,9 @@ class InstantiationData(object): def environment_files(self): return self.data.get(self.PARAM_ENVIRONMENT_FILES, None) + def files_container(self): + return self.data.get(self.PARAM_FILES_CONTAINER, None) + def args(self): """Get any additional arguments supplied by the user.""" params = self.data.items() @@ -369,8 +374,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files() - ) + environment_files=data.environment_files(), + files_container=data.files_container()) formatted_stack = stacks_view.format_stack(req, result) return {'stack': formatted_stack} @@ -403,7 +408,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) formatted_stack = stacks_view.format_stack( req, @@ -486,7 +492,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) raise exc.HTTPAccepted() @@ -507,7 +514,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) raise exc.HTTPAccepted() @@ -535,7 +543,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) return {'resource_changes': changes} @@ -555,7 +564,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) return {'resource_changes': changes} @@ -616,6 +626,7 @@ class StackController(object): data.environment(), files=data.files(), environment_files=data.environment_files(), + files_container=data.files_container(), show_nested=show_nested, ignorable_errors=ignorable_errors) diff --git a/heat/common/exception.py b/heat/common/exception.py index 02d817b08d..a60981339f 100644 --- a/heat/common/exception.py +++ b/heat/common/exception.py @@ -478,6 +478,10 @@ class RequestLimitExceeded(HeatException): msg_fmt = _('Request limit exceeded: %(message)s') +class DownloadLimitExceeded(HeatException): + msg_fmt = _('Permissible download limit exceeded: %(message)s') + + class StackResourceLimitExceeded(HeatException): msg_fmt = _('Maximum resources per stack exceeded.') diff --git a/heat/engine/clients/os/swift.py b/heat/engine/clients/os/swift.py index edddaeb18e..7167bb0866 100644 --- a/heat/engine/clients/os/swift.py +++ b/heat/engine/clients/os/swift.py @@ -18,12 +18,15 @@ import logging import random import time +from oslo_config import cfg import six from six.moves.urllib import parse from swiftclient import client as sc from swiftclient import exceptions from swiftclient import utils as swiftclient_utils +from heat.common import exception +from heat.common.i18n import _ from heat.engine.clients import client_plugin IN_PROGRESS = 'in progress' @@ -137,3 +140,38 @@ class SwiftClientPlugin(client_plugin.ClientPlugin): # according to RFC 2616, all HTTP time headers must be # in GMT time, so create an offset-naive UTC datetime return datetime.datetime(*pd) + + def get_files_from_container(self, files_container, files_to_skip=None): + """Gets the file contents from a container. + + Get the file contents from the container in a files map. A list + of files to skip can also be specified and those would not be + downloaded from swift. + """ + client = self.client() + files = {} + + if files_to_skip is None: + files_to_skip = [] + + try: + headers, objects = client.get_container(files_container) + bytes_used = headers.get('x-container-bytes-used', 0) + if bytes_used > cfg.CONF.max_json_body_size: + msg = _("Total size of files to download (%(size)s bytes) " + "exceeds maximum allowed (%(limit)s bytes).") % { + 'size': bytes_used, + 'limit': cfg.CONF.max_json_body_size} + raise exception.DownloadLimitExceeded(message=msg) + for obj in objects: + file_name = obj['name'] + if file_name not in files_to_skip: + contents = client.get_object(files_container, file_name)[1] + files[file_name] = contents + except exceptions.ClientException as cex: + raise exception.NotFound(_('Could not fetch files from ' + 'container %(container)s, ' + 'reason: %(reason)s.') % + {'container': files_container, + 'reason': six.text_type(cex)}) + return files diff --git a/heat/engine/service.py b/heat/engine/service.py index 5156ed165a..b528f243a9 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -56,6 +56,7 @@ from heat.engine import stack_lock from heat.engine import stk_defn from heat.engine import support from heat.engine import template as templatem +from heat.engine import template_files from heat.engine import update from heat.engine import worker from heat.objects import event as event_object @@ -306,7 +307,7 @@ class EngineService(service.ServiceBase): by the RPC caller. """ - RPC_API_VERSION = '1.35' + RPC_API_VERSION = '1.36' def __init__(self, host, topic): resources.initialise() @@ -656,8 +657,9 @@ class EngineService(service.ServiceBase): def _parse_template_and_validate_stack(self, cnxt, stack_name, template, params, files, environment_files, - args, owner_id=None, - nested_depth=0, user_creds_id=None, + files_container, args, + owner_id=None, nested_depth=0, + user_creds_id=None, stack_user_project_id=None, convergence=False, parent_resource_name=None, @@ -680,9 +682,12 @@ class EngineService(service.ServiceBase): if template_id is not None: tmpl = templatem.Template.load(cnxt, template_id) else: + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) tmpl = templatem.Template(template, files=files) - env_util.merge_environments(environment_files, files, params, - tmpl.all_param_schemata(files)) + env_util.merge_environments(environment_files, files, + params, tmpl.all_param_schemata(files)) tmpl.env = environment.Environment(params) self._validate_new_stack(cnxt, stack_name, tmpl) @@ -706,7 +711,7 @@ class EngineService(service.ServiceBase): @context.request_context def preview_stack(self, cnxt, stack_name, template, params, files, - args, environment_files=None): + args, environment_files=None, files_container=None): """Simulate a new stack using the provided template. Note that at this stage the template has already been fetched from the @@ -721,6 +726,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name """ LOG.info('previewing stack %s', stack_name) @@ -732,6 +738,7 @@ class EngineService(service.ServiceBase): params, files, environment_files, + files_container, args, convergence=conv_eng) @@ -740,7 +747,8 @@ class EngineService(service.ServiceBase): @context.request_context def create_stack(self, cnxt, stack_name, template, params, files, args, environment_files=None, - owner_id=None, nested_depth=0, user_creds_id=None, + files_container=None, owner_id=None, + nested_depth=0, user_creds_id=None, stack_user_project_id=None, parent_resource_name=None, template_id=None): """Create a new stack using the template provided. @@ -757,6 +765,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name :param owner_id: parent stack ID for nested stacks, only expected when called from another heat-engine (not a user option) :param nested_depth: the nested depth for nested stacks, only expected @@ -788,9 +797,9 @@ class EngineService(service.ServiceBase): stack = self._parse_template_and_validate_stack( cnxt, stack_name, template, params, files, environment_files, - args, owner_id, nested_depth, user_creds_id, - stack_user_project_id, convergence, parent_resource_name, - template_id) + files_container, args, owner_id, nested_depth, + user_creds_id, stack_user_project_id, convergence, + parent_resource_name, template_id) stack_id = stack.store() if cfg.CONF.reauthentication_auth_method == 'trusts': @@ -817,7 +826,8 @@ class EngineService(service.ServiceBase): def _prepare_stack_updates(self, cnxt, current_stack, template, params, environment_files, - files, args, template_id=None): + files, files_container, + args, template_id=None): """Return the current and updated stack for a given transition. Changes *will not* be persisted, this is a helper method for @@ -866,10 +876,13 @@ class EngineService(service.ServiceBase): raise exception.NotSupported(feature=msg) new_files = current_stack.t.files + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) new_files.update(files or {}) tmpl = templatem.Template(new_template, files=new_files) - env_util.merge_environments(environment_files, files, params, - tmpl.all_param_schemata(files)) + env_util.merge_environments(environment_files, new_files, + params, tmpl.all_param_schemata(files)) existing_env = current_stack.env.env_as_dict() existing_params = existing_env[env_fmt.PARAMETERS] clear_params = set(args.get(rpc_api.PARAM_CLEAR_PARAMETERS, [])) @@ -888,8 +901,12 @@ class EngineService(service.ServiceBase): if template_id is not None: tmpl = templatem.Template.load(cnxt, template_id) else: + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) tmpl = templatem.Template(template, files=files) - env_util.merge_environments(environment_files, files, params, + env_util.merge_environments(environment_files, + files, params, tmpl.all_param_schemata(files)) tmpl.env = environment.Environment(params) @@ -932,7 +949,8 @@ class EngineService(service.ServiceBase): @context.request_context def update_stack(self, cnxt, stack_identity, template, params, - files, args, environment_files=None, template_id=None): + files, args, environment_files=None, + files_container=None, template_id=None): """Update an existing stack based on the provided template and params. Note that at this stage the template has already been fetched from the @@ -947,6 +965,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name :param template_id: the ID of a pre-stored template in the DB """ # Get the database representation of the existing stack @@ -970,7 +989,8 @@ class EngineService(service.ServiceBase): tmpl, current_stack, updated_stack = self._prepare_stack_updates( cnxt, current_stack, template, params, - environment_files, files, args, template_id) + environment_files, files, files_container, + args, template_id) if current_stack.convergence: current_stack.thread_group_mgr = self.thread_group_mgr @@ -990,7 +1010,8 @@ class EngineService(service.ServiceBase): @context.request_context def preview_update_stack(self, cnxt, stack_identity, template, params, - files, args, environment_files=None): + files, args, environment_files=None, + files_container=None): """Shows the resources that would be updated. The preview_update_stack method shows the resources that would be @@ -1013,7 +1034,7 @@ class EngineService(service.ServiceBase): tmpl, current_stack, updated_stack = self._prepare_stack_updates( cnxt, current_stack, template, params, - environment_files, files, args) + environment_files, files, files_container, args) update_task = update.StackUpdate(current_stack, updated_stack, None) @@ -1172,8 +1193,8 @@ class EngineService(service.ServiceBase): @context.request_context def validate_template(self, cnxt, template, params=None, files=None, - environment_files=None, show_nested=False, - ignorable_errors=None): + environment_files=None, files_container=None, + show_nested=False, ignorable_errors=None): """Check the validity of a template. Checks, so far as we can, that a template is valid, and returns @@ -1187,6 +1208,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name :param show_nested: if True, any nested templates will be checked :param ignorable_errors: List of error_code to be ignored as part of validation @@ -1203,10 +1225,12 @@ class EngineService(service.ServiceBase): msg = (_("Invalid codes in ignore_errors : %s") % list(invalid_codes)) return webob.exc.HTTPBadRequest(explanation=msg) - + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) tmpl = templatem.Template(template, files=files) - env_util.merge_environments(environment_files, files, params, - tmpl.all_param_schemata(files)) + env_util.merge_environments(environment_files, files, + params, tmpl.all_param_schemata(files)) tmpl.env = environment.Environment(params) try: self._validate_template(cnxt, tmpl) diff --git a/heat/engine/template_files.py b/heat/engine/template_files.py index 844a5fbe48..0e7f6e77e6 100644 --- a/heat/engine/template_files.py +++ b/heat/engine/template_files.py @@ -16,6 +16,7 @@ import six import weakref from heat.common import context +from heat.common import exception from heat.common.i18n import _ from heat.db.sqlalchemy import api as db_api from heat.objects import raw_template_files @@ -134,3 +135,21 @@ class TemplateFiles(collections.Mapping): new_files = files self.files_id = None # not persisted yet self.files = ReadOnlyDict(new_files) + + +def get_files_from_container(cnxt, files_container, files=None): + + if files is None: + files = {} + else: + files = files.copy() + + swift_plugin = cnxt.clients.client_plugin('swift') + + if not swift_plugin: + raise exception.ClientNotAvailable(client_name='swift') + + new_files = swift_plugin.get_files_from_container(files_container, + list(files.keys())) + new_files.update(files) + return new_files diff --git a/heat/rpc/client.py b/heat/rpc/client.py index 5e833f93e0..b2f4a45990 100644 --- a/heat/rpc/client.py +++ b/heat/rpc/client.py @@ -61,6 +61,7 @@ class EngineClient(object): and list_software_configs 1.34 - Add migrate_convergence_1 call 1.35 - Add with_condition to list_template_functions + 1.36 - Add files_container to create/update/preview/validate """ BASE_RPC_API_VERSION = '1.0' @@ -226,7 +227,7 @@ class EngineClient(object): version='1.20') def preview_stack(self, ctxt, stack_name, template, params, files, - args, environment_files=None): + args, environment_files=None, files_container=None): """Simulates a new stack using the provided template. Note that at this stage the template has already been fetched from the @@ -241,17 +242,19 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self.call(ctxt, self.make_msg('preview_stack', stack_name=stack_name, template=template, params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args), - version='1.23') + version='1.36') def create_stack(self, ctxt, stack_name, template, params, files, - args, environment_files=None): + args, environment_files=None, files_container=None): """Creates a new stack using the template provided. Note that at this stage the template has already been fetched from the @@ -266,12 +269,14 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self._create_stack(ctxt, stack_name, template, params, files, - args, environment_files=environment_files) + args, environment_files=environment_files, + files_container=files_container) def _create_stack(self, ctxt, stack_name, template, params, files, - args, environment_files=None, + args, environment_files=None, files_container=None, owner_id=None, nested_depth=0, user_creds_id=None, stack_user_project_id=None, parent_resource_name=None, template_id=None): @@ -292,16 +297,18 @@ class EngineClient(object): template=template, params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args, owner_id=owner_id, nested_depth=nested_depth, user_creds_id=user_creds_id, stack_user_project_id=stack_user_project_id, parent_resource_name=parent_resource_name, template_id=template_id), - version='1.29') + version='1.36') def update_stack(self, ctxt, stack_identity, template, params, - files, args, environment_files=None): + files, args, environment_files=None, + files_container=None): """Updates an existing stack based on the provided template and params. Note that at this stage the template has already been fetched from the @@ -316,14 +323,16 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self._update_stack(ctxt, stack_identity, template, params, files, args, - environment_files=environment_files) + environment_files=environment_files, + files_container=files_container) def _update_stack(self, ctxt, stack_identity, template, params, files, args, environment_files=None, - template_id=None): + files_container=None, template_id=None): """Internal interface for engine-to-engine communication via RPC. Allows an additional option which should not be exposed to users via @@ -338,12 +347,14 @@ class EngineClient(object): params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args, template_id=template_id), - version='1.29') + version='1.36') def preview_update_stack(self, ctxt, stack_identity, template, params, - files, args, environment_files=None): + files, args, environment_files=None, + files_container=None): """Returns the resources that would be changed in an update. Based on the provided template and parameters. @@ -359,6 +370,7 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self.call(ctxt, self.make_msg('preview_update_stack', @@ -367,13 +379,14 @@ class EngineClient(object): params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args, ), - version='1.23') + version='1.36') def validate_template(self, ctxt, template, params=None, files=None, - environment_files=None, show_nested=False, - ignorable_errors=None): + environment_files=None, files_container=None, + show_nested=False, ignorable_errors=None): """Uses the stack parser to check the validity of a template. :param ctxt: RPC context. @@ -382,6 +395,7 @@ class EngineClient(object): :param files: files referenced from the environment/template. :param environment_files: ordered list of environment file names included in the files dict + :param files_container: name of swift container :param show_nested: if True nested templates will be validated :param ignorable_errors: List of error_code to be ignored as part of validation @@ -393,8 +407,9 @@ class EngineClient(object): files=files, show_nested=show_nested, environment_files=environment_files, + files_container=files_container, ignorable_errors=ignorable_errors), - version='1.24') + version='1.36') def authenticated_to_backend(self, ctxt): """Validate the credentials in the RPC context. diff --git a/heat/tests/api/cfn/test_api_cfn_v1.py b/heat/tests/api/cfn/test_api_cfn_v1.py index abd2fbddfa..7b444f4dd6 100644 --- a/heat/tests/api/cfn/test_api_cfn_v1.py +++ b/heat/tests/api/cfn/test_api_cfn_v1.py @@ -546,6 +546,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -553,7 +554,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_rollback(self): @@ -592,6 +593,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -599,7 +601,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_onfailure_true(self): @@ -638,6 +640,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -645,7 +648,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_onfailure_false_delete(self): @@ -674,6 +677,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -681,7 +685,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) expected = { @@ -730,6 +734,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -737,7 +742,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_onfailure_err(self): @@ -913,9 +918,10 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'template_id': None}), - version='1.29' + version='1.36' )], self.m_call.call_args_list) def test_cancel_update(self): @@ -1091,9 +1097,10 @@ class CfnStackControllerTest(common.HeatTestCase): dummy_req.context, ('validate_template', {'template': json_template, 'params': None, 'files': None, 'environment_files': None, + 'files_container': None, 'show_nested': False, 'ignorable_errors': None}), - version='1.24' + version='1.36' ) def test_delete(self): diff --git a/heat/tests/api/openstack_v1/test_stacks.py b/heat/tests/api/openstack_v1/test_stacks.py index 2fa3ebc604..b0c51799af 100644 --- a/heat/tests/api/openstack_v1/test_stacks.py +++ b/heat/tests/api/openstack_v1/test_stacks.py @@ -756,6 +756,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': ['foo.yaml'], + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -763,7 +764,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_with_tags(self, mock_enforce): @@ -803,6 +804,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'tags': ['tag1', 'tag2']}, 'owner_id': None, 'nested_depth': 0, @@ -810,7 +812,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_adopt(self, mock_enforce): @@ -868,6 +870,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'adopt_stack_data': str(adopt_data)}, 'owner_id': None, @@ -876,7 +879,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_adopt_timeout_not_int(self, mock_enforce): @@ -957,6 +960,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {'my.yaml': 'This is the file contents.'}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -964,7 +968,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_err_rpcerr(self, mock_enforce): @@ -1025,6 +1029,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -1032,7 +1037,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) self.assertEqual(3, mock_call.call_count) @@ -1072,6 +1077,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -1079,7 +1085,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_timeout_not_int(self, mock_enforce): @@ -1158,6 +1164,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -1165,7 +1172,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_err_stack_bad_reqest(self, mock_enforce): @@ -1235,8 +1242,9 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'tags': ['tag1', 'tag2']}}), - version='1.23' + version='1.36' ) self.assertEqual({'stack': 'formatted_stack_preview'}, response) @@ -1280,8 +1288,9 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}}), - version='1.23' + version='1.36' ) def test_preview_update_stack_patch(self, mock_enforce): @@ -1321,9 +1330,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}}), - version='1.23' + version='1.36' ) @mock.patch.object(rpc_client.EngineClient, 'call') @@ -1369,9 +1379,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): u'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_lookup(self, mock_enforce): @@ -1829,9 +1840,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_tags(self, mock_enforce): @@ -1870,9 +1882,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'tags': ['tag1', 'tag2']}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_bad_name(self, mock_enforce): @@ -1914,9 +1927,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): u'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_timeout_not_int(self, mock_enforce): @@ -1999,10 +2013,11 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_existing_parameters(self, mock_enforce): @@ -2039,10 +2054,11 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_existing_parameters_with_tags(self, mock_enforce): @@ -2080,11 +2096,12 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30, 'tags': ['tag1', 'tag2']}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_patched_existing_parameters(self, mock_enforce): @@ -2122,10 +2139,11 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_patch_timeout_not_int(self, mock_enforce): @@ -2189,11 +2207,12 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'clear_parameters': clear_params, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_patched_and_default_parameters( @@ -2234,11 +2253,12 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'clear_parameters': clear_params, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_delete(self, mock_enforce): @@ -2398,9 +2418,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'show_nested': False, 'ignorable_errors': None}), - version='1.24' + version='1.36' ) def test_validate_template_error(self, mock_enforce): @@ -2428,9 +2449,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'show_nested': False, 'ignorable_errors': None}), - version='1.24' + version='1.36' ) def test_validate_err_denied_policy(self, mock_enforce): diff --git a/heat/tests/engine/service/test_service_engine.py b/heat/tests/engine/service/test_service_engine.py index 494a2dbe92..a66e8e0ded 100644 --- a/heat/tests/engine/service/test_service_engine.py +++ b/heat/tests/engine/service/test_service_engine.py @@ -39,7 +39,7 @@ class ServiceEngineTest(common.HeatTestCase): def test_make_sure_rpc_version(self): self.assertEqual( - '1.35', + '1.36', service.EngineService.RPC_API_VERSION, ('RPC version is changed, please update this test to new version ' 'and make sure additional test cases are added for RPC APIs ' diff --git a/heat/tests/engine/service/test_stack_create.py b/heat/tests/engine/service/test_stack_create.py index 88c5971bf9..29b99355e4 100644 --- a/heat/tests/engine/service/test_stack_create.py +++ b/heat/tests/engine/service/test_stack_create.py @@ -10,16 +10,19 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + import mock from oslo_config import cfg from oslo_messaging.rpc import dispatcher from oslo_service import threadgroup import six +from swiftclient import exceptions from heat.common import environment_util as env_util from heat.common import exception from heat.engine.clients.os import glance from heat.engine.clients.os import nova +from heat.engine.clients.os import swift from heat.engine import environment from heat.engine import properties from heat.engine.resources.aws.ec2 import instance as instances @@ -43,7 +46,8 @@ class StackCreateTest(common.HeatTestCase): @mock.patch.object(threadgroup, 'ThreadGroup') @mock.patch.object(stack.Stack, 'validate') def _test_stack_create(self, stack_name, mock_validate, mock_tg, - environment_files=None): + environment_files=None, files_container=None, + error=False): mock_tg.return_value = tools.DummyThreadGroup() params = {'foo': 'bar'} @@ -51,29 +55,44 @@ class StackCreateTest(common.HeatTestCase): stk = tools.get_stack(stack_name, self.ctx) + files = None + if files_container: + files = {'/env/test.yaml': "{'resource_registry': {}}"} + mock_tmpl = self.patchobject(templatem, 'Template', return_value=stk.t) mock_env = self.patchobject(environment, 'Environment', return_value=stk.env) mock_stack = self.patchobject(stack, 'Stack', return_value=stk) mock_merge = self.patchobject(env_util, 'merge_environments') - result = self.man.create_stack(self.ctx, stack_name, - template, params, None, {}, - environment_files=environment_files) - self.assertEqual(stk.identifier(), result) - self.assertIsInstance(result, dict) - self.assertTrue(result['stack_id']) - - mock_tmpl.assert_called_once_with(template, files=None) - mock_env.assert_called_once_with(params) - mock_stack.assert_called_once_with( - self.ctx, stack_name, stk.t, owner_id=None, nested_depth=0, - user_creds_id=None, stack_user_project_id=None, - convergence=cfg.CONF.convergence_engine, parent_resource=None) - - if environment_files: - mock_merge.assert_called_once_with(environment_files, None, - params, mock.ANY) - mock_validate.assert_called_once_with() + if not error: + result = self.man.create_stack(self.ctx, stack_name, + template, params, None, {}, + environment_files=environment_files, + files_container=files_container) + self.assertEqual(stk.identifier(), result) + self.assertIsInstance(result, dict) + self.assertTrue(result['stack_id']) + mock_tmpl.assert_called_once_with(template, files=files) + mock_env.assert_called_once_with(params) + mock_stack.assert_called_once_with( + self.ctx, stack_name, stk.t, owner_id=None, nested_depth=0, + user_creds_id=None, stack_user_project_id=None, + convergence=cfg.CONF.convergence_engine, parent_resource=None) + if environment_files: + mock_merge.assert_called_once_with(environment_files, files, + params, mock.ANY) + mock_validate.assert_called_once_with() + else: + ex = self.assertRaises(dispatcher.ExpectedException, + self.man.create_stack, + self.ctx, stack_name, + template, params, None, {}, + environment_files=environment_files, + files_container=files_container) + self.assertEqual(exception.NotFound, ex.exc_info[0]) + self.assertIn('Could not fetch files from container ' + 'test_container, reason: error.', + six.text_type(ex.exc_info[1])) def test_stack_create(self): stack_name = 'service_create_test_stack' @@ -85,6 +104,41 @@ class StackCreateTest(common.HeatTestCase): self._test_stack_create(stack_name, environment_files=environment_files) + def test_stack_create_with_files_container(self): + stack_name = 'env_files_test_stack' + environment_files = ['env_1', 'env_2'] + files_container = 'test_container' + fake_get_object = (None, "{'resource_registry': {}}") + fake_get_container = ({'x-container-bytes-used': 100}, + [{'name': '/env/test.yaml'}]) + mock_client = mock.Mock() + mock_client.get_object.return_value = fake_get_object + mock_client.get_container.return_value = fake_get_container + self.patchobject(swift.SwiftClientPlugin, '_create', + return_value=mock_client) + self._test_stack_create(stack_name, + environment_files=environment_files, + files_container=files_container) + mock_client.get_container.assert_called_with(files_container) + mock_client.get_object.assert_called_with(files_container, + '/env/test.yaml') + + def test_stack_create_with_container_notfound_swift(self): + stack_name = 'env_files_test_stack' + environment_files = ['env_1', 'env_2'] + files_container = 'test_container' + mock_client = mock.Mock() + mock_client.get_container.side_effect = exceptions.ClientException( + 'error') + self.patchobject(swift.SwiftClientPlugin, '_create', + return_value=mock_client) + self._test_stack_create(stack_name, + environment_files=environment_files, + files_container=files_container, + error=True) + mock_client.get_container.assert_called_with(files_container) + mock_client.get_object.assert_not_called() + def test_stack_create_equals_max_per_tenant(self): cfg.CONF.set_override('max_stacks_per_tenant', 1) stack_name = 'service_create_test_stack_equals_max' diff --git a/heat/tests/engine/service/test_stack_update.py b/heat/tests/engine/service/test_stack_update.py index 8848da0779..8c71b7ce17 100644 --- a/heat/tests/engine/service/test_stack_update.py +++ b/heat/tests/engine/service/test_stack_update.py @@ -26,6 +26,7 @@ from heat.common import template_format from heat.db.sqlalchemy import api as db_api from heat.engine.clients.os import glance from heat.engine.clients.os import nova +from heat.engine.clients.os import swift from heat.engine import environment from heat.engine import resource from heat.engine import service @@ -103,9 +104,9 @@ class ServiceStackUpdateTest(common.HeatTestCase): mock_load.assert_called_once_with(self.ctx, stack=s) mock_validate.assert_called_once_with() - def test_stack_update_with_environment_files(self): + def _test_stack_update_with_environment_files(self, stack_name, + files_container=None): # Setup - stack_name = 'service_update_env_files_stack' params = {} template = '{ "Template": "data" }' old_stack = tools.get_stack(stack_name, self.ctx) @@ -126,17 +127,42 @@ class ServiceStackUpdateTest(common.HeatTestCase): mock_merge = self.patchobject(env_util, 'merge_environments') + files = None + if files_container: + files = {'/env/test.yaml': "{'resource_registry': {}}"} + # Test environment_files = ['env_1'] self.man.update_stack(self.ctx, old_stack.identifier(), template, params, None, {rpc_api.PARAM_CONVERGE: False}, - environment_files=environment_files) - + environment_files=environment_files, + files_container=files_container) # Verify - mock_merge.assert_called_once_with(environment_files, None, + mock_merge.assert_called_once_with(environment_files, files, params, mock.ANY) + def test_stack_update_with_environment_files(self): + stack_name = 'service_update_env_files_stack' + self._test_stack_update_with_environment_files(stack_name) + + def test_stack_update_with_files_container(self): + stack_name = 'env_files_test_stack' + files_container = 'test_container' + fake_get_object = (None, "{'resource_registry': {}}") + fake_get_container = ({'x-container-bytes-used': 100}, + [{'name': '/env/test.yaml'}]) + mock_client = mock.Mock() + mock_client.get_object.return_value = fake_get_object + mock_client.get_container.return_value = fake_get_container + self.patchobject(swift.SwiftClientPlugin, '_create', + return_value=mock_client) + self._test_stack_update_with_environment_files( + stack_name, files_container=files_container) + mock_client.get_container.assert_called_with(files_container) + mock_client.get_object.assert_called_with(files_container, + '/env/test.yaml') + def test_stack_update_nested(self): stack_name = 'service_update_nested_test_stack' parent_stack = tools.get_stack(stack_name + '_parent', self.ctx) @@ -348,20 +374,20 @@ resources: # update keep old tags _, _, updated_stack = self.man._prepare_stack_updates( - self.ctx, stk, t, {}, None, None, api_args, None) + self.ctx, stk, t, {}, None, None, None, api_args, None) self.assertEqual(['tag1'], updated_stack.tags) # with new tags api_args[rpc_api.STACK_TAGS] = ['tag2'] _, _, updated_stack = self.man._prepare_stack_updates( - self.ctx, stk, t, {}, None, None, api_args, None) + self.ctx, stk, t, {}, None, None, None, api_args, None) self.assertEqual(['tag2'], updated_stack.tags) # with no PARAM_EXISTING flag and no tags del api_args[rpc_api.PARAM_EXISTING] del api_args[rpc_api.STACK_TAGS] _, _, updated_stack = self.man._prepare_stack_updates( - self.ctx, stk, t, {}, None, None, api_args, None) + self.ctx, stk, t, {}, None, None, None, api_args, None) self.assertIsNone(updated_stack.tags) def test_stack_update_existing_registry(self): diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index 809999aeb8..671c65a2c5 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -1324,13 +1324,15 @@ class StackServiceTest(common.HeatTestCase): # get parameters from adopt stack data which doesn't have it. args = {"adopt_stack_data": '''{}'''} self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, None, args) + self.ctx, 'stack_name', template, {}, {}, + None, None, args) args = {"adopt_stack_data": '''{ "environment": {} }'''} self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, None, args) + self.ctx, 'stack_name', template, {}, {}, + None, None, args) def test_parse_adopt_stack_data_with_parameters(self): cfg.CONF.set_override('enable_stack_adopt', True) @@ -1355,7 +1357,8 @@ class StackServiceTest(common.HeatTestCase): } }}'''} stack = self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, None, args) + self.ctx, 'stack_name', template, {}, {}, + None, None, args) self.assertEqual(1, stack.parameters['volsize']) @mock.patch('heat.engine.service.ThreadGroupManager', diff --git a/heat/tests/test_rpc_client.py b/heat/tests/test_rpc_client.py index 7aa7fc3518..7bf414216c 100644 --- a/heat/tests/test_rpc_client.py +++ b/heat/tests/test_rpc_client.py @@ -165,6 +165,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={u'a_file': u'the contents'}, environment_files=['foo.yaml'], + files_container=None, args={'timeout_mins': u'30'}) def test_create_stack(self): @@ -173,6 +174,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={u'a_file': u'the contents'}, environment_files=['foo.yaml'], + files_container=None, args={'timeout_mins': u'30'}) call_kwargs = copy.deepcopy(kwargs) call_kwargs['owner_id'] = None @@ -191,6 +193,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={}, environment_files=['foo.yaml'], + files_container=None, args=mock.ANY) call_kwargs = copy.deepcopy(kwargs) call_kwargs['template_id'] = None @@ -206,6 +209,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={}, environment_files=['foo.yaml'], + files_container=None, args=mock.ANY) def test_get_template(self): @@ -226,6 +230,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'Egg': u'spam'}, files=None, environment_files=['foo.yaml'], + files_container=None, ignorable_errors=None, show_nested=False, version='1.24')