diff --git a/heat/api/middleware/fault.py b/heat/api/middleware/fault.py index 2e977393a7..a190ff84d6 100644 --- a/heat/api/middleware/fault.py +++ b/heat/api/middleware/fault.py @@ -82,6 +82,7 @@ class FaultWrapper(wsgi.Middleware): 'MissingCredentialError': webob.exc.HTTPBadRequest, 'UserParameterMissing': webob.exc.HTTPBadRequest, 'RequestLimitExceeded': webob.exc.HTTPBadRequest, + 'DownloadLimitExceeded': webob.exc.HTTPBadRequest, 'Invalid': webob.exc.HTTPBadRequest, 'ResourcePropertyConflict': webob.exc.HTTPBadRequest, 'PropertyUnspecifiedError': webob.exc.HTTPBadRequest, diff --git a/heat/api/openstack/v1/stacks.py b/heat/api/openstack/v1/stacks.py index cbbbded381..d144ea1972 100644 --- a/heat/api/openstack/v1/stacks.py +++ b/heat/api/openstack/v1/stacks.py @@ -50,6 +50,7 @@ class InstantiationData(object): PARAM_ENVIRONMENT, PARAM_FILES, PARAM_ENVIRONMENT_FILES, + PARAM_FILES_CONTAINER ) = ( 'stack_name', 'template', @@ -58,6 +59,7 @@ class InstantiationData(object): 'environment', 'files', 'environment_files', + 'files_container' ) def __init__(self, data, patch=False): @@ -157,6 +159,9 @@ class InstantiationData(object): def environment_files(self): return self.data.get(self.PARAM_ENVIRONMENT_FILES, None) + def files_container(self): + return self.data.get(self.PARAM_FILES_CONTAINER, None) + def args(self): """Get any additional arguments supplied by the user.""" params = self.data.items() @@ -369,8 +374,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files() - ) + environment_files=data.environment_files(), + files_container=data.files_container()) formatted_stack = stacks_view.format_stack(req, result) return {'stack': formatted_stack} @@ -403,7 +408,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) formatted_stack = stacks_view.format_stack( req, @@ -486,7 +492,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) raise exc.HTTPAccepted() @@ -507,7 +514,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) raise exc.HTTPAccepted() @@ -535,7 +543,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) return {'resource_changes': changes} @@ -555,7 +564,8 @@ class StackController(object): data.environment(), data.files(), args, - environment_files=data.environment_files()) + environment_files=data.environment_files(), + files_container=data.files_container()) return {'resource_changes': changes} @@ -616,6 +626,7 @@ class StackController(object): data.environment(), files=data.files(), environment_files=data.environment_files(), + files_container=data.files_container(), show_nested=show_nested, ignorable_errors=ignorable_errors) diff --git a/heat/common/exception.py b/heat/common/exception.py index 02d817b08d..a60981339f 100644 --- a/heat/common/exception.py +++ b/heat/common/exception.py @@ -478,6 +478,10 @@ class RequestLimitExceeded(HeatException): msg_fmt = _('Request limit exceeded: %(message)s') +class DownloadLimitExceeded(HeatException): + msg_fmt = _('Permissible download limit exceeded: %(message)s') + + class StackResourceLimitExceeded(HeatException): msg_fmt = _('Maximum resources per stack exceeded.') diff --git a/heat/engine/clients/os/swift.py b/heat/engine/clients/os/swift.py index edddaeb18e..7167bb0866 100644 --- a/heat/engine/clients/os/swift.py +++ b/heat/engine/clients/os/swift.py @@ -18,12 +18,15 @@ import logging import random import time +from oslo_config import cfg import six from six.moves.urllib import parse from swiftclient import client as sc from swiftclient import exceptions from swiftclient import utils as swiftclient_utils +from heat.common import exception +from heat.common.i18n import _ from heat.engine.clients import client_plugin IN_PROGRESS = 'in progress' @@ -137,3 +140,38 @@ class SwiftClientPlugin(client_plugin.ClientPlugin): # according to RFC 2616, all HTTP time headers must be # in GMT time, so create an offset-naive UTC datetime return datetime.datetime(*pd) + + def get_files_from_container(self, files_container, files_to_skip=None): + """Gets the file contents from a container. + + Get the file contents from the container in a files map. A list + of files to skip can also be specified and those would not be + downloaded from swift. + """ + client = self.client() + files = {} + + if files_to_skip is None: + files_to_skip = [] + + try: + headers, objects = client.get_container(files_container) + bytes_used = headers.get('x-container-bytes-used', 0) + if bytes_used > cfg.CONF.max_json_body_size: + msg = _("Total size of files to download (%(size)s bytes) " + "exceeds maximum allowed (%(limit)s bytes).") % { + 'size': bytes_used, + 'limit': cfg.CONF.max_json_body_size} + raise exception.DownloadLimitExceeded(message=msg) + for obj in objects: + file_name = obj['name'] + if file_name not in files_to_skip: + contents = client.get_object(files_container, file_name)[1] + files[file_name] = contents + except exceptions.ClientException as cex: + raise exception.NotFound(_('Could not fetch files from ' + 'container %(container)s, ' + 'reason: %(reason)s.') % + {'container': files_container, + 'reason': six.text_type(cex)}) + return files diff --git a/heat/engine/service.py b/heat/engine/service.py index 5156ed165a..b528f243a9 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -56,6 +56,7 @@ from heat.engine import stack_lock from heat.engine import stk_defn from heat.engine import support from heat.engine import template as templatem +from heat.engine import template_files from heat.engine import update from heat.engine import worker from heat.objects import event as event_object @@ -306,7 +307,7 @@ class EngineService(service.ServiceBase): by the RPC caller. """ - RPC_API_VERSION = '1.35' + RPC_API_VERSION = '1.36' def __init__(self, host, topic): resources.initialise() @@ -656,8 +657,9 @@ class EngineService(service.ServiceBase): def _parse_template_and_validate_stack(self, cnxt, stack_name, template, params, files, environment_files, - args, owner_id=None, - nested_depth=0, user_creds_id=None, + files_container, args, + owner_id=None, nested_depth=0, + user_creds_id=None, stack_user_project_id=None, convergence=False, parent_resource_name=None, @@ -680,9 +682,12 @@ class EngineService(service.ServiceBase): if template_id is not None: tmpl = templatem.Template.load(cnxt, template_id) else: + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) tmpl = templatem.Template(template, files=files) - env_util.merge_environments(environment_files, files, params, - tmpl.all_param_schemata(files)) + env_util.merge_environments(environment_files, files, + params, tmpl.all_param_schemata(files)) tmpl.env = environment.Environment(params) self._validate_new_stack(cnxt, stack_name, tmpl) @@ -706,7 +711,7 @@ class EngineService(service.ServiceBase): @context.request_context def preview_stack(self, cnxt, stack_name, template, params, files, - args, environment_files=None): + args, environment_files=None, files_container=None): """Simulate a new stack using the provided template. Note that at this stage the template has already been fetched from the @@ -721,6 +726,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name """ LOG.info('previewing stack %s', stack_name) @@ -732,6 +738,7 @@ class EngineService(service.ServiceBase): params, files, environment_files, + files_container, args, convergence=conv_eng) @@ -740,7 +747,8 @@ class EngineService(service.ServiceBase): @context.request_context def create_stack(self, cnxt, stack_name, template, params, files, args, environment_files=None, - owner_id=None, nested_depth=0, user_creds_id=None, + files_container=None, owner_id=None, + nested_depth=0, user_creds_id=None, stack_user_project_id=None, parent_resource_name=None, template_id=None): """Create a new stack using the template provided. @@ -757,6 +765,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name :param owner_id: parent stack ID for nested stacks, only expected when called from another heat-engine (not a user option) :param nested_depth: the nested depth for nested stacks, only expected @@ -788,9 +797,9 @@ class EngineService(service.ServiceBase): stack = self._parse_template_and_validate_stack( cnxt, stack_name, template, params, files, environment_files, - args, owner_id, nested_depth, user_creds_id, - stack_user_project_id, convergence, parent_resource_name, - template_id) + files_container, args, owner_id, nested_depth, + user_creds_id, stack_user_project_id, convergence, + parent_resource_name, template_id) stack_id = stack.store() if cfg.CONF.reauthentication_auth_method == 'trusts': @@ -817,7 +826,8 @@ class EngineService(service.ServiceBase): def _prepare_stack_updates(self, cnxt, current_stack, template, params, environment_files, - files, args, template_id=None): + files, files_container, + args, template_id=None): """Return the current and updated stack for a given transition. Changes *will not* be persisted, this is a helper method for @@ -866,10 +876,13 @@ class EngineService(service.ServiceBase): raise exception.NotSupported(feature=msg) new_files = current_stack.t.files + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) new_files.update(files or {}) tmpl = templatem.Template(new_template, files=new_files) - env_util.merge_environments(environment_files, files, params, - tmpl.all_param_schemata(files)) + env_util.merge_environments(environment_files, new_files, + params, tmpl.all_param_schemata(files)) existing_env = current_stack.env.env_as_dict() existing_params = existing_env[env_fmt.PARAMETERS] clear_params = set(args.get(rpc_api.PARAM_CLEAR_PARAMETERS, [])) @@ -888,8 +901,12 @@ class EngineService(service.ServiceBase): if template_id is not None: tmpl = templatem.Template.load(cnxt, template_id) else: + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) tmpl = templatem.Template(template, files=files) - env_util.merge_environments(environment_files, files, params, + env_util.merge_environments(environment_files, + files, params, tmpl.all_param_schemata(files)) tmpl.env = environment.Environment(params) @@ -932,7 +949,8 @@ class EngineService(service.ServiceBase): @context.request_context def update_stack(self, cnxt, stack_identity, template, params, - files, args, environment_files=None, template_id=None): + files, args, environment_files=None, + files_container=None, template_id=None): """Update an existing stack based on the provided template and params. Note that at this stage the template has already been fetched from the @@ -947,6 +965,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name :param template_id: the ID of a pre-stored template in the DB """ # Get the database representation of the existing stack @@ -970,7 +989,8 @@ class EngineService(service.ServiceBase): tmpl, current_stack, updated_stack = self._prepare_stack_updates( cnxt, current_stack, template, params, - environment_files, files, args, template_id) + environment_files, files, files_container, + args, template_id) if current_stack.convergence: current_stack.thread_group_mgr = self.thread_group_mgr @@ -990,7 +1010,8 @@ class EngineService(service.ServiceBase): @context.request_context def preview_update_stack(self, cnxt, stack_identity, template, params, - files, args, environment_files=None): + files, args, environment_files=None, + files_container=None): """Shows the resources that would be updated. The preview_update_stack method shows the resources that would be @@ -1013,7 +1034,7 @@ class EngineService(service.ServiceBase): tmpl, current_stack, updated_stack = self._prepare_stack_updates( cnxt, current_stack, template, params, - environment_files, files, args) + environment_files, files, files_container, args) update_task = update.StackUpdate(current_stack, updated_stack, None) @@ -1172,8 +1193,8 @@ class EngineService(service.ServiceBase): @context.request_context def validate_template(self, cnxt, template, params=None, files=None, - environment_files=None, show_nested=False, - ignorable_errors=None): + environment_files=None, files_container=None, + show_nested=False, ignorable_errors=None): """Check the validity of a template. Checks, so far as we can, that a template is valid, and returns @@ -1187,6 +1208,7 @@ class EngineService(service.ServiceBase): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: optional swift container name :param show_nested: if True, any nested templates will be checked :param ignorable_errors: List of error_code to be ignored as part of validation @@ -1203,10 +1225,12 @@ class EngineService(service.ServiceBase): msg = (_("Invalid codes in ignore_errors : %s") % list(invalid_codes)) return webob.exc.HTTPBadRequest(explanation=msg) - + if files_container: + files = template_files.get_files_from_container( + cnxt, files_container, files) tmpl = templatem.Template(template, files=files) - env_util.merge_environments(environment_files, files, params, - tmpl.all_param_schemata(files)) + env_util.merge_environments(environment_files, files, + params, tmpl.all_param_schemata(files)) tmpl.env = environment.Environment(params) try: self._validate_template(cnxt, tmpl) diff --git a/heat/engine/template_files.py b/heat/engine/template_files.py index 844a5fbe48..0e7f6e77e6 100644 --- a/heat/engine/template_files.py +++ b/heat/engine/template_files.py @@ -16,6 +16,7 @@ import six import weakref from heat.common import context +from heat.common import exception from heat.common.i18n import _ from heat.db.sqlalchemy import api as db_api from heat.objects import raw_template_files @@ -134,3 +135,21 @@ class TemplateFiles(collections.Mapping): new_files = files self.files_id = None # not persisted yet self.files = ReadOnlyDict(new_files) + + +def get_files_from_container(cnxt, files_container, files=None): + + if files is None: + files = {} + else: + files = files.copy() + + swift_plugin = cnxt.clients.client_plugin('swift') + + if not swift_plugin: + raise exception.ClientNotAvailable(client_name='swift') + + new_files = swift_plugin.get_files_from_container(files_container, + list(files.keys())) + new_files.update(files) + return new_files diff --git a/heat/rpc/client.py b/heat/rpc/client.py index 5e833f93e0..b2f4a45990 100644 --- a/heat/rpc/client.py +++ b/heat/rpc/client.py @@ -61,6 +61,7 @@ class EngineClient(object): and list_software_configs 1.34 - Add migrate_convergence_1 call 1.35 - Add with_condition to list_template_functions + 1.36 - Add files_container to create/update/preview/validate """ BASE_RPC_API_VERSION = '1.0' @@ -226,7 +227,7 @@ class EngineClient(object): version='1.20') def preview_stack(self, ctxt, stack_name, template, params, files, - args, environment_files=None): + args, environment_files=None, files_container=None): """Simulates a new stack using the provided template. Note that at this stage the template has already been fetched from the @@ -241,17 +242,19 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self.call(ctxt, self.make_msg('preview_stack', stack_name=stack_name, template=template, params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args), - version='1.23') + version='1.36') def create_stack(self, ctxt, stack_name, template, params, files, - args, environment_files=None): + args, environment_files=None, files_container=None): """Creates a new stack using the template provided. Note that at this stage the template has already been fetched from the @@ -266,12 +269,14 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self._create_stack(ctxt, stack_name, template, params, files, - args, environment_files=environment_files) + args, environment_files=environment_files, + files_container=files_container) def _create_stack(self, ctxt, stack_name, template, params, files, - args, environment_files=None, + args, environment_files=None, files_container=None, owner_id=None, nested_depth=0, user_creds_id=None, stack_user_project_id=None, parent_resource_name=None, template_id=None): @@ -292,16 +297,18 @@ class EngineClient(object): template=template, params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args, owner_id=owner_id, nested_depth=nested_depth, user_creds_id=user_creds_id, stack_user_project_id=stack_user_project_id, parent_resource_name=parent_resource_name, template_id=template_id), - version='1.29') + version='1.36') def update_stack(self, ctxt, stack_identity, template, params, - files, args, environment_files=None): + files, args, environment_files=None, + files_container=None): """Updates an existing stack based on the provided template and params. Note that at this stage the template has already been fetched from the @@ -316,14 +323,16 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self._update_stack(ctxt, stack_identity, template, params, files, args, - environment_files=environment_files) + environment_files=environment_files, + files_container=files_container) def _update_stack(self, ctxt, stack_identity, template, params, files, args, environment_files=None, - template_id=None): + files_container=None, template_id=None): """Internal interface for engine-to-engine communication via RPC. Allows an additional option which should not be exposed to users via @@ -338,12 +347,14 @@ class EngineClient(object): params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args, template_id=template_id), - version='1.29') + version='1.36') def preview_update_stack(self, ctxt, stack_identity, template, params, - files, args, environment_files=None): + files, args, environment_files=None, + files_container=None): """Returns the resources that would be changed in an update. Based on the provided template and parameters. @@ -359,6 +370,7 @@ class EngineClient(object): :param environment_files: optional ordered list of environment file names included in the files dict :type environment_files: list or None + :param files_container: name of swift container """ return self.call(ctxt, self.make_msg('preview_update_stack', @@ -367,13 +379,14 @@ class EngineClient(object): params=params, files=files, environment_files=environment_files, + files_container=files_container, args=args, ), - version='1.23') + version='1.36') def validate_template(self, ctxt, template, params=None, files=None, - environment_files=None, show_nested=False, - ignorable_errors=None): + environment_files=None, files_container=None, + show_nested=False, ignorable_errors=None): """Uses the stack parser to check the validity of a template. :param ctxt: RPC context. @@ -382,6 +395,7 @@ class EngineClient(object): :param files: files referenced from the environment/template. :param environment_files: ordered list of environment file names included in the files dict + :param files_container: name of swift container :param show_nested: if True nested templates will be validated :param ignorable_errors: List of error_code to be ignored as part of validation @@ -393,8 +407,9 @@ class EngineClient(object): files=files, show_nested=show_nested, environment_files=environment_files, + files_container=files_container, ignorable_errors=ignorable_errors), - version='1.24') + version='1.36') def authenticated_to_backend(self, ctxt): """Validate the credentials in the RPC context. diff --git a/heat/tests/api/cfn/test_api_cfn_v1.py b/heat/tests/api/cfn/test_api_cfn_v1.py index abd2fbddfa..7b444f4dd6 100644 --- a/heat/tests/api/cfn/test_api_cfn_v1.py +++ b/heat/tests/api/cfn/test_api_cfn_v1.py @@ -546,6 +546,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -553,7 +554,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_rollback(self): @@ -592,6 +593,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -599,7 +601,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_onfailure_true(self): @@ -638,6 +640,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -645,7 +648,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_onfailure_false_delete(self): @@ -674,6 +677,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -681,7 +685,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) expected = { @@ -730,6 +734,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'owner_id': None, 'nested_depth': 0, @@ -737,7 +742,7 @@ class CfnStackControllerTest(common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_onfailure_err(self): @@ -913,9 +918,10 @@ class CfnStackControllerTest(common.HeatTestCase): 'params': engine_parms, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': engine_args, 'template_id': None}), - version='1.29' + version='1.36' )], self.m_call.call_args_list) def test_cancel_update(self): @@ -1091,9 +1097,10 @@ class CfnStackControllerTest(common.HeatTestCase): dummy_req.context, ('validate_template', {'template': json_template, 'params': None, 'files': None, 'environment_files': None, + 'files_container': None, 'show_nested': False, 'ignorable_errors': None}), - version='1.24' + version='1.36' ) def test_delete(self): diff --git a/heat/tests/api/openstack_v1/test_stacks.py b/heat/tests/api/openstack_v1/test_stacks.py index 2fa3ebc604..b0c51799af 100644 --- a/heat/tests/api/openstack_v1/test_stacks.py +++ b/heat/tests/api/openstack_v1/test_stacks.py @@ -756,6 +756,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': ['foo.yaml'], + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -763,7 +764,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_with_tags(self, mock_enforce): @@ -803,6 +804,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'tags': ['tag1', 'tag2']}, 'owner_id': None, 'nested_depth': 0, @@ -810,7 +812,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_adopt(self, mock_enforce): @@ -868,6 +870,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'adopt_stack_data': str(adopt_data)}, 'owner_id': None, @@ -876,7 +879,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_adopt_timeout_not_int(self, mock_enforce): @@ -957,6 +960,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {'my.yaml': 'This is the file contents.'}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -964,7 +968,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_err_rpcerr(self, mock_enforce): @@ -1025,6 +1029,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -1032,7 +1037,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) self.assertEqual(3, mock_call.call_count) @@ -1072,6 +1077,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -1079,7 +1085,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_timeout_not_int(self, mock_enforce): @@ -1158,6 +1164,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'owner_id': None, 'nested_depth': 0, @@ -1165,7 +1172,7 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'parent_resource_name': None, 'stack_user_project_id': None, 'template_id': None}), - version='1.29' + version='1.36' ) def test_create_err_stack_bad_reqest(self, mock_enforce): @@ -1235,8 +1242,9 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'tags': ['tag1', 'tag2']}}), - version='1.23' + version='1.36' ) self.assertEqual({'stack': 'formatted_stack_preview'}, response) @@ -1280,8 +1288,9 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}}), - version='1.23' + version='1.36' ) def test_preview_update_stack_patch(self, mock_enforce): @@ -1321,9 +1330,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}}), - version='1.23' + version='1.36' ) @mock.patch.object(rpc_client.EngineClient, 'call') @@ -1369,9 +1379,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): u'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_lookup(self, mock_enforce): @@ -1829,9 +1840,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_tags(self, mock_enforce): @@ -1870,9 +1882,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30, 'tags': ['tag1', 'tag2']}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_bad_name(self, mock_enforce): @@ -1914,9 +1927,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): u'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_timeout_not_int(self, mock_enforce): @@ -1999,10 +2013,11 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_existing_parameters(self, mock_enforce): @@ -2039,10 +2054,11 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_existing_parameters_with_tags(self, mock_enforce): @@ -2080,11 +2096,12 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30, 'tags': ['tag1', 'tag2']}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_patched_existing_parameters(self, mock_enforce): @@ -2122,10 +2139,11 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_patch_timeout_not_int(self, mock_enforce): @@ -2189,11 +2207,12 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'clear_parameters': clear_params, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_update_with_patched_and_default_parameters( @@ -2234,11 +2253,12 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'args': {rpc_api.PARAM_EXISTING: True, 'clear_parameters': clear_params, 'timeout_mins': 30}, 'template_id': None}), - version='1.29' + version='1.36' ) def test_delete(self, mock_enforce): @@ -2398,9 +2418,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'show_nested': False, 'ignorable_errors': None}), - version='1.24' + version='1.36' ) def test_validate_template_error(self, mock_enforce): @@ -2428,9 +2449,10 @@ class StackControllerTest(tools.ControllerTest, common.HeatTestCase): 'resource_registry': {}}, 'files': {}, 'environment_files': None, + 'files_container': None, 'show_nested': False, 'ignorable_errors': None}), - version='1.24' + version='1.36' ) def test_validate_err_denied_policy(self, mock_enforce): diff --git a/heat/tests/engine/service/test_service_engine.py b/heat/tests/engine/service/test_service_engine.py index 494a2dbe92..a66e8e0ded 100644 --- a/heat/tests/engine/service/test_service_engine.py +++ b/heat/tests/engine/service/test_service_engine.py @@ -39,7 +39,7 @@ class ServiceEngineTest(common.HeatTestCase): def test_make_sure_rpc_version(self): self.assertEqual( - '1.35', + '1.36', service.EngineService.RPC_API_VERSION, ('RPC version is changed, please update this test to new version ' 'and make sure additional test cases are added for RPC APIs ' diff --git a/heat/tests/engine/service/test_stack_create.py b/heat/tests/engine/service/test_stack_create.py index 88c5971bf9..29b99355e4 100644 --- a/heat/tests/engine/service/test_stack_create.py +++ b/heat/tests/engine/service/test_stack_create.py @@ -10,16 +10,19 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + import mock from oslo_config import cfg from oslo_messaging.rpc import dispatcher from oslo_service import threadgroup import six +from swiftclient import exceptions from heat.common import environment_util as env_util from heat.common import exception from heat.engine.clients.os import glance from heat.engine.clients.os import nova +from heat.engine.clients.os import swift from heat.engine import environment from heat.engine import properties from heat.engine.resources.aws.ec2 import instance as instances @@ -43,7 +46,8 @@ class StackCreateTest(common.HeatTestCase): @mock.patch.object(threadgroup, 'ThreadGroup') @mock.patch.object(stack.Stack, 'validate') def _test_stack_create(self, stack_name, mock_validate, mock_tg, - environment_files=None): + environment_files=None, files_container=None, + error=False): mock_tg.return_value = tools.DummyThreadGroup() params = {'foo': 'bar'} @@ -51,29 +55,44 @@ class StackCreateTest(common.HeatTestCase): stk = tools.get_stack(stack_name, self.ctx) + files = None + if files_container: + files = {'/env/test.yaml': "{'resource_registry': {}}"} + mock_tmpl = self.patchobject(templatem, 'Template', return_value=stk.t) mock_env = self.patchobject(environment, 'Environment', return_value=stk.env) mock_stack = self.patchobject(stack, 'Stack', return_value=stk) mock_merge = self.patchobject(env_util, 'merge_environments') - result = self.man.create_stack(self.ctx, stack_name, - template, params, None, {}, - environment_files=environment_files) - self.assertEqual(stk.identifier(), result) - self.assertIsInstance(result, dict) - self.assertTrue(result['stack_id']) - - mock_tmpl.assert_called_once_with(template, files=None) - mock_env.assert_called_once_with(params) - mock_stack.assert_called_once_with( - self.ctx, stack_name, stk.t, owner_id=None, nested_depth=0, - user_creds_id=None, stack_user_project_id=None, - convergence=cfg.CONF.convergence_engine, parent_resource=None) - - if environment_files: - mock_merge.assert_called_once_with(environment_files, None, - params, mock.ANY) - mock_validate.assert_called_once_with() + if not error: + result = self.man.create_stack(self.ctx, stack_name, + template, params, None, {}, + environment_files=environment_files, + files_container=files_container) + self.assertEqual(stk.identifier(), result) + self.assertIsInstance(result, dict) + self.assertTrue(result['stack_id']) + mock_tmpl.assert_called_once_with(template, files=files) + mock_env.assert_called_once_with(params) + mock_stack.assert_called_once_with( + self.ctx, stack_name, stk.t, owner_id=None, nested_depth=0, + user_creds_id=None, stack_user_project_id=None, + convergence=cfg.CONF.convergence_engine, parent_resource=None) + if environment_files: + mock_merge.assert_called_once_with(environment_files, files, + params, mock.ANY) + mock_validate.assert_called_once_with() + else: + ex = self.assertRaises(dispatcher.ExpectedException, + self.man.create_stack, + self.ctx, stack_name, + template, params, None, {}, + environment_files=environment_files, + files_container=files_container) + self.assertEqual(exception.NotFound, ex.exc_info[0]) + self.assertIn('Could not fetch files from container ' + 'test_container, reason: error.', + six.text_type(ex.exc_info[1])) def test_stack_create(self): stack_name = 'service_create_test_stack' @@ -85,6 +104,41 @@ class StackCreateTest(common.HeatTestCase): self._test_stack_create(stack_name, environment_files=environment_files) + def test_stack_create_with_files_container(self): + stack_name = 'env_files_test_stack' + environment_files = ['env_1', 'env_2'] + files_container = 'test_container' + fake_get_object = (None, "{'resource_registry': {}}") + fake_get_container = ({'x-container-bytes-used': 100}, + [{'name': '/env/test.yaml'}]) + mock_client = mock.Mock() + mock_client.get_object.return_value = fake_get_object + mock_client.get_container.return_value = fake_get_container + self.patchobject(swift.SwiftClientPlugin, '_create', + return_value=mock_client) + self._test_stack_create(stack_name, + environment_files=environment_files, + files_container=files_container) + mock_client.get_container.assert_called_with(files_container) + mock_client.get_object.assert_called_with(files_container, + '/env/test.yaml') + + def test_stack_create_with_container_notfound_swift(self): + stack_name = 'env_files_test_stack' + environment_files = ['env_1', 'env_2'] + files_container = 'test_container' + mock_client = mock.Mock() + mock_client.get_container.side_effect = exceptions.ClientException( + 'error') + self.patchobject(swift.SwiftClientPlugin, '_create', + return_value=mock_client) + self._test_stack_create(stack_name, + environment_files=environment_files, + files_container=files_container, + error=True) + mock_client.get_container.assert_called_with(files_container) + mock_client.get_object.assert_not_called() + def test_stack_create_equals_max_per_tenant(self): cfg.CONF.set_override('max_stacks_per_tenant', 1) stack_name = 'service_create_test_stack_equals_max' diff --git a/heat/tests/engine/service/test_stack_update.py b/heat/tests/engine/service/test_stack_update.py index 8848da0779..8c71b7ce17 100644 --- a/heat/tests/engine/service/test_stack_update.py +++ b/heat/tests/engine/service/test_stack_update.py @@ -26,6 +26,7 @@ from heat.common import template_format from heat.db.sqlalchemy import api as db_api from heat.engine.clients.os import glance from heat.engine.clients.os import nova +from heat.engine.clients.os import swift from heat.engine import environment from heat.engine import resource from heat.engine import service @@ -103,9 +104,9 @@ class ServiceStackUpdateTest(common.HeatTestCase): mock_load.assert_called_once_with(self.ctx, stack=s) mock_validate.assert_called_once_with() - def test_stack_update_with_environment_files(self): + def _test_stack_update_with_environment_files(self, stack_name, + files_container=None): # Setup - stack_name = 'service_update_env_files_stack' params = {} template = '{ "Template": "data" }' old_stack = tools.get_stack(stack_name, self.ctx) @@ -126,17 +127,42 @@ class ServiceStackUpdateTest(common.HeatTestCase): mock_merge = self.patchobject(env_util, 'merge_environments') + files = None + if files_container: + files = {'/env/test.yaml': "{'resource_registry': {}}"} + # Test environment_files = ['env_1'] self.man.update_stack(self.ctx, old_stack.identifier(), template, params, None, {rpc_api.PARAM_CONVERGE: False}, - environment_files=environment_files) - + environment_files=environment_files, + files_container=files_container) # Verify - mock_merge.assert_called_once_with(environment_files, None, + mock_merge.assert_called_once_with(environment_files, files, params, mock.ANY) + def test_stack_update_with_environment_files(self): + stack_name = 'service_update_env_files_stack' + self._test_stack_update_with_environment_files(stack_name) + + def test_stack_update_with_files_container(self): + stack_name = 'env_files_test_stack' + files_container = 'test_container' + fake_get_object = (None, "{'resource_registry': {}}") + fake_get_container = ({'x-container-bytes-used': 100}, + [{'name': '/env/test.yaml'}]) + mock_client = mock.Mock() + mock_client.get_object.return_value = fake_get_object + mock_client.get_container.return_value = fake_get_container + self.patchobject(swift.SwiftClientPlugin, '_create', + return_value=mock_client) + self._test_stack_update_with_environment_files( + stack_name, files_container=files_container) + mock_client.get_container.assert_called_with(files_container) + mock_client.get_object.assert_called_with(files_container, + '/env/test.yaml') + def test_stack_update_nested(self): stack_name = 'service_update_nested_test_stack' parent_stack = tools.get_stack(stack_name + '_parent', self.ctx) @@ -348,20 +374,20 @@ resources: # update keep old tags _, _, updated_stack = self.man._prepare_stack_updates( - self.ctx, stk, t, {}, None, None, api_args, None) + self.ctx, stk, t, {}, None, None, None, api_args, None) self.assertEqual(['tag1'], updated_stack.tags) # with new tags api_args[rpc_api.STACK_TAGS] = ['tag2'] _, _, updated_stack = self.man._prepare_stack_updates( - self.ctx, stk, t, {}, None, None, api_args, None) + self.ctx, stk, t, {}, None, None, None, api_args, None) self.assertEqual(['tag2'], updated_stack.tags) # with no PARAM_EXISTING flag and no tags del api_args[rpc_api.PARAM_EXISTING] del api_args[rpc_api.STACK_TAGS] _, _, updated_stack = self.man._prepare_stack_updates( - self.ctx, stk, t, {}, None, None, api_args, None) + self.ctx, stk, t, {}, None, None, None, api_args, None) self.assertIsNone(updated_stack.tags) def test_stack_update_existing_registry(self): diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index 809999aeb8..671c65a2c5 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -1324,13 +1324,15 @@ class StackServiceTest(common.HeatTestCase): # get parameters from adopt stack data which doesn't have it. args = {"adopt_stack_data": '''{}'''} self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, None, args) + self.ctx, 'stack_name', template, {}, {}, + None, None, args) args = {"adopt_stack_data": '''{ "environment": {} }'''} self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, None, args) + self.ctx, 'stack_name', template, {}, {}, + None, None, args) def test_parse_adopt_stack_data_with_parameters(self): cfg.CONF.set_override('enable_stack_adopt', True) @@ -1355,7 +1357,8 @@ class StackServiceTest(common.HeatTestCase): } }}'''} stack = self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, None, args) + self.ctx, 'stack_name', template, {}, {}, + None, None, args) self.assertEqual(1, stack.parameters['volsize']) @mock.patch('heat.engine.service.ThreadGroupManager', diff --git a/heat/tests/test_rpc_client.py b/heat/tests/test_rpc_client.py index 7aa7fc3518..7bf414216c 100644 --- a/heat/tests/test_rpc_client.py +++ b/heat/tests/test_rpc_client.py @@ -165,6 +165,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={u'a_file': u'the contents'}, environment_files=['foo.yaml'], + files_container=None, args={'timeout_mins': u'30'}) def test_create_stack(self): @@ -173,6 +174,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={u'a_file': u'the contents'}, environment_files=['foo.yaml'], + files_container=None, args={'timeout_mins': u'30'}) call_kwargs = copy.deepcopy(kwargs) call_kwargs['owner_id'] = None @@ -191,6 +193,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={}, environment_files=['foo.yaml'], + files_container=None, args=mock.ANY) call_kwargs = copy.deepcopy(kwargs) call_kwargs['template_id'] = None @@ -206,6 +209,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'InstanceType': u'm1.xlarge'}, files={}, environment_files=['foo.yaml'], + files_container=None, args=mock.ANY) def test_get_template(self): @@ -226,6 +230,7 @@ class EngineRpcAPITestCase(common.HeatTestCase): params={u'Egg': u'spam'}, files=None, environment_files=['foo.yaml'], + files_container=None, ignorable_errors=None, show_nested=False, version='1.24')