diff --git a/heat/engine/service.py b/heat/engine/service.py index c22a0001f5..67968b9f43 100644 --- a/heat/engine/service.py +++ b/heat/engine/service.py @@ -69,6 +69,8 @@ from heat.objects import watch_data from heat.objects import watch_rule from heat.rpc import api as rpc_api from heat.rpc import worker_api as rpc_worker_api +from heatclient.common import environment_format +from heatclient.common import template_utils cfg.CONF.import_opt('engine_life_check_timeout', 'heat.common.config') cfg.CONF.import_opt('max_resources_per_stack', 'heat.common.config') @@ -621,7 +623,8 @@ class EngineService(service.Service): raise exception.RequestLimitExceeded(message=message) def _parse_template_and_validate_stack(self, cnxt, stack_name, template, - params, files, args, owner_id=None, + params, files, environment_files, + args, owner_id=None, nested_depth=0, user_creds_id=None, stack_user_project_id=None, convergence=False, @@ -641,6 +644,7 @@ class EngineService(service.Service): new_params.update(params.get(rpc_api.STACK_PARAMETERS, {})) params[rpc_api.STACK_PARAMETERS] = new_params + self._merge_environments(environment_files, files, params) env = environment.Environment(params) tmpl = templatem.Template(template, files=files, env=env) @@ -662,6 +666,30 @@ class EngineService(service.Service): env.registry.log_resource_info(prefix=stack_name) return stack + @staticmethod + def _merge_environments(environment_files, files, params): + """Merges environment files into the stack input parameters. + + If a list of environment files have been specified, this call will + pull the contents of each from the files dict, parse them as + environments, and merge them into the stack input params. This + behavior is the same as earlier versions of the Heat client that + performed this params population client-side. + + :param environment_files: ordered names of the environment files + found in the files dict + :type environment_files: list or None + :param files: mapping of stack filenames to contents + :type files: dict + :param params: parameters describing the stack + :type dict: + """ + if environment_files: + for filename in environment_files: + raw_env = files[filename] + parsed_env = environment_format.parse(raw_env) + template_utils.deep_update(params, parsed_env) + @context.request_context def preview_stack(self, cnxt, stack_name, template, params, files, args, environment_files=None): @@ -689,6 +717,7 @@ class EngineService(service.Service): template, params, files, + environment_files, args, convergence=conv_eng) @@ -751,9 +780,9 @@ class EngineService(service.Service): convergence = cfg.CONF.convergence_engine stack = self._parse_template_and_validate_stack( - cnxt, stack_name, template, params, files, args, owner_id, - nested_depth, user_creds_id, stack_user_project_id, convergence, - parent_resource_name) + cnxt, stack_name, template, params, files, environment_files, + args, owner_id, nested_depth, user_creds_id, + stack_user_project_id, convergence, parent_resource_name) self.resource_enforcer.enforce_stack(stack) stack_id = stack.store() @@ -786,6 +815,7 @@ class EngineService(service.Service): :param files: Files referenced from the template :param args: Request parameters/args passed from API """ + # Now parse the template and any parameters for the updated # stack definition. If PARAM_EXISTING is specified, we merge # any environment provided into the existing one and attempt @@ -877,6 +907,9 @@ class EngineService(service.Service): names included in the files dict :type environment_files: list or None """ + # Handle server-side environment file resolution + self._merge_environments(environment_files, files, params) + # Get the database representation of the existing stack db_stack = self._get_stack(cnxt, stack_identity) LOG.info(_LI('Updating stack %s'), db_stack.name) @@ -930,6 +963,9 @@ class EngineService(service.Service): Note that at this stage the template has already been fetched from the heat-api process if using a template-url. """ + # Handle server-side environment file resolution + self._merge_environments(environment_files, files, params) + # Get the database representation of the existing stack db_stack = self._get_stack(cnxt, stack_identity) LOG.info(_LI('Previewing update of stack %s'), db_stack.name) @@ -1104,6 +1140,7 @@ class EngineService(service.Service): service_check_defer = True + self._merge_environments(environment_files, files, params) env = environment.Environment(params) tmpl = templatem.Template(template, files=files, env=env) try: diff --git a/heat/tests/engine/service/test_service_engine.py b/heat/tests/engine/service/test_service_engine.py index 18f338cfdd..2bafbc10f3 100644 --- a/heat/tests/engine/service/test_service_engine.py +++ b/heat/tests/engine/service/test_service_engine.py @@ -411,3 +411,50 @@ class ServiceEngineTest(common.HeatTestCase): self.eng.start() self.assertEqual(cfg.CONF.executor_thread_pool_size, cfg.CONF.database.max_overflow) + + def test_merge_environments(self): + # Setup + params = {'parameters': { + 'p0': 'CORRECT', + 'p1': 'INCORRECT', + 'p2': 'INCORRECT'} + } + env_1 = ''' + {'parameters' : { + 'p1': 'CORRECT', + 'p2': 'INCORRECT-ENV-1', + }}''' + env_2 = ''' + {'parameters': { + 'p2': 'CORRECT' + }}''' + + files = {'env_1': env_1, 'env_2': env_2} + environment_files = ['env_1', 'env_2'] + + # Test + self.eng._merge_environments(environment_files, files, params) + + # Verify + expected = {'parameters': { + 'p0': 'CORRECT', + 'p1': 'CORRECT', + 'p2': 'CORRECT', + }} + self.assertEqual(expected, params) + + def test_merge_environments_no_env_files(self): + params = {'parameters': {'p0': 'CORRECT'}} + env_1 = ''' + {'parameters' : { + 'p0': 'INCORRECT', + }}''' + + files = {'env_1': env_1} + + # Test - Should ignore env_1 in files + self.eng._merge_environments(None, files, params) + + # Verify + expected = {'parameters': {'p0': 'CORRECT'}} + self.assertEqual(expected, params) diff --git a/heat/tests/engine/service/test_stack_create.py b/heat/tests/engine/service/test_stack_create.py index 78374c623c..4390dd8d01 100644 --- a/heat/tests/engine/service/test_stack_create.py +++ b/heat/tests/engine/service/test_stack_create.py @@ -41,7 +41,8 @@ class StackCreateTest(common.HeatTestCase): @mock.patch.object(threadgroup, 'ThreadGroup') @mock.patch.object(stack.Stack, 'validate') - def _test_stack_create(self, stack_name, mock_validate, mock_tg): + def _test_stack_create(self, stack_name, mock_validate, mock_tg, + environment_files=None): mock_tg.return_value = tools.DummyThreadGroup() params = {'foo': 'bar'} @@ -53,8 +54,10 @@ class StackCreateTest(common.HeatTestCase): mock_env = self.patchobject(environment, 'Environment', return_value=stk.env) mock_stack = self.patchobject(stack, 'Stack', return_value=stk) + mock_merge = self.patchobject(self.man, '_merge_environments') result = self.man.create_stack(self.ctx, stack_name, - template, params, None, {}) + template, params, None, {}, + environment_files=environment_files) self.assertEqual(stk.identifier(), result) self.assertIsInstance(result, dict) self.assertTrue(result['stack_id']) @@ -67,12 +70,21 @@ class StackCreateTest(common.HeatTestCase): stack_user_project_id=None, convergence=False, parent_resource=None) + + if environment_files: + mock_merge.assert_called_once_with(environment_files, None, params) mock_validate.assert_called_once_with() def test_stack_create(self): stack_name = 'service_create_test_stack' self._test_stack_create(stack_name) + def test_stack_create_with_environment_files(self): + stack_name = 'env_files_test_stack' + environment_files = ['env_1', 'env_2'] + self._test_stack_create(stack_name, + environment_files=environment_files) + def test_stack_create_equals_max_per_tenant(self): cfg.CONF.set_override('max_stacks_per_tenant', 1) stack_name = 'service_create_test_stack_equals_max' diff --git a/heat/tests/engine/service/test_stack_update.py b/heat/tests/engine/service/test_stack_update.py index 782d511dfc..22974258cd 100644 --- a/heat/tests/engine/service/test_stack_update.py +++ b/heat/tests/engine/service/test_stack_update.py @@ -96,6 +96,37 @@ class ServiceStackUpdateTest(common.HeatTestCase): mock_load.assert_called_once_with(self.ctx, stack=s) mock_validate.assert_called_once_with() + def test_stack_update_with_environment_files(self): + # Setup + stack_name = 'service_update_env_files_stack' + params = {} + template = '{ "Template": "data" }' + old_stack = tools.get_stack(stack_name, self.ctx) + sid = old_stack.store() + old_stack.set_stack_user_project_id('1234') + stack_object.Stack.get_by_id(self.ctx, sid) + + stk = tools.get_stack(stack_name, self.ctx) + + # prepare mocks + self.patchobject(stack, 'Stack', return_value=stk) + self.patchobject(stack.Stack, 'load', return_value=old_stack) + self.patchobject(templatem, 'Template', return_value=stk.t) + self.patchobject(environment, 'Environment', return_value=stk.env) + self.patchobject(stk, 'validate', return_value=None) + self.patchobject(grevent, 'Event', return_value=mock.Mock()) + + mock_merge = self.patchobject(self.man, '_merge_environments') + + # Test + environment_files = ['env_1'] + self.man.update_stack(self.ctx, old_stack.identifier(), + template, params, None, {}, + environment_files=environment_files) + + # Verify + mock_merge.assert_called_once_with(environment_files, None, params) + def test_stack_update_existing_parameters(self): # Use a template with existing parameters, then update the stack # with a template containing additional parameters and ensure all @@ -712,7 +743,8 @@ resources: self.man = service.EngineService('a-host', 'a-topic') self.man.thread_group_mgr = tools.DummyThreadGroupManager() - def _test_stack_update_preview(self, orig_template, new_template): + def _test_stack_update_preview(self, orig_template, new_template, + environment_files=None): stack_name = 'service_update_test_stack_preview' params = {'foo': 'bar'} old_stack = tools.get_stack(stack_name, self.ctx, @@ -731,6 +763,7 @@ resources: mock_env = self.patchobject(environment, 'Environment', return_value=stk.env) mock_validate = self.patchobject(stk, 'validate', return_value=None) + mock_merge = self.patchobject(self.man, '_merge_environments') # Patch _resolve_all_attributes or it tries to call novaclient self.patchobject(resource.Resource, '_resolve_all_attributes', @@ -738,10 +771,13 @@ resources: # do preview_update_stack api_args = {'timeout_mins': 60} - result = self.man.preview_update_stack(self.ctx, - old_stack.identifier(), - new_template, params, None, - api_args) + result = self.man.preview_update_stack( + self.ctx, + old_stack.identifier(), + new_template, params, None, + api_args, + environment_files=environment_files) + # assertions mock_stack.assert_called_once_with( self.ctx, stk.name, stk.t, convergence=False, @@ -757,6 +793,9 @@ resources: mock_env.assert_called_once_with(params) mock_validate.assert_called_once_with() + if environment_files: + mock_merge.assert_called_once_with(environment_files, None, params) + return result def test_stack_update_preview_added_unchanged(self): @@ -824,3 +863,13 @@ resources: for section in empty_sections: section_contents = [x for x in result[section]] self.assertEqual([], section_contents) + + def test_stack_update_preview_with_environment_files(self): + # Setup + environment_files = ['env_1'] + + # Test + self._test_stack_update_preview(self.old_tmpl, self.new_tmpl, + environment_files=environment_files) + + # Assertions done in _test_stack_update_preview diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index aad16c84f7..a2a266973b 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -1123,7 +1123,7 @@ class StackServiceTest(common.HeatTestCase): self.assertEqual(0, len(sl)) - def _preview_stack(self): + def _preview_stack(self, environment_files=None): res._register_class('GenericResource1', generic_rsrc.GenericResource) res._register_class('GenericResource2', generic_rsrc.GenericResource) @@ -1138,7 +1138,8 @@ class StackServiceTest(common.HeatTestCase): 'SampleResource2': {'Type': 'GenericResource2'}}} return self.eng.preview_stack(self.ctx, stack_name, tpl, - params, files, args) + params, files, args, + environment_files=environment_files) def test_preview_stack_returns_a_stack(self): stack = self._preview_stack() @@ -1181,6 +1182,17 @@ class StackServiceTest(common.HeatTestCase): self._preview_stack) self.assertEqual(exception.StackValidationFailed, ex.exc_info[0]) + @mock.patch.object(service.EngineService, '_merge_environments') + def test_preview_environment_files(self, mock_merge): + # Setup + environment_files = ['env_1'] + + # Test + self._preview_stack(environment_files=environment_files) + + # Verify + mock_merge.assert_called_once_with(environment_files, None, {}) + @mock.patch.object(stack_object.Stack, 'get_by_name') def test_validate_new_stack_checks_existing_stack(self, mock_stack_get): mock_stack_get.return_value = 'existing_db_stack' @@ -1389,13 +1401,13 @@ class StackServiceTest(common.HeatTestCase): # get parameters from adopt stack data which doesn't have it. args = {"adopt_stack_data": '''{}'''} self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, args) + self.ctx, 'stack_name', template, {}, {}, None, args) args = {"adopt_stack_data": '''{ "environment": {} }'''} self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, args) + self.ctx, 'stack_name', template, {}, {}, None, args) def test_parse_adopt_stack_data_with_parameters(self): cfg.CONF.set_override('enable_stack_adopt', True) @@ -1420,5 +1432,5 @@ class StackServiceTest(common.HeatTestCase): } }}'''} stack = self.eng._parse_template_and_validate_stack( - self.ctx, 'stack_name', template, {}, {}, args) + self.ctx, 'stack_name', template, {}, {}, None, args) self.assertEqual(1, stack.parameters['volsize']) diff --git a/heat_integrationtests/common/test.py b/heat_integrationtests/common/test.py index 8fc5c602a3..c54febe95a 100644 --- a/heat_integrationtests/common/test.py +++ b/heat_integrationtests/common/test.py @@ -475,7 +475,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios, def stack_create(self, stack_name=None, template=None, files=None, parameters=None, environment=None, tags=None, expected_status='CREATE_COMPLETE', - disable_rollback=True, enable_cleanup=True): + disable_rollback=True, enable_cleanup=True, + environment_files=None): name = stack_name or self._stack_rand_name() templ = template or self.template templ_files = files or {} @@ -488,7 +489,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios, disable_rollback=disable_rollback, parameters=params, environment=env, - tags=tags + tags=tags, + environment_files=environment_files ) if expected_status not in ['ROLLBACK_COMPLETE'] and enable_cleanup: self.addCleanup(self._stack_delete, name) diff --git a/heat_integrationtests/functional/test_env_merge.py b/heat_integrationtests/functional/test_env_merge.py new file mode 100644 index 0000000000..5e222b85f6 --- /dev/null +++ b/heat_integrationtests/functional/test_env_merge.py @@ -0,0 +1,95 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from heat_integrationtests.functional import functional_base + + +TEMPLATE = ''' + heat_template_version: 2015-04-30 + parameters: + p0: + type: string + default: CORRECT + p1: + type: string + default: INCORRECT + p2: + type: string + default: INCORRECT + resources: + r1: + type: test::R1 + r2: + type: test::R2 + r3a: + type: test::R3 + r3b: + type: test::R3 +''' + +ENV_1 = ''' + parameters: + p1: CORRECT + p2: INCORRECT-E1 + resource_registry: + test::R1: OS::Heat::RandomString + test::R2: BROKEN + test::R3: OS::Heat::None +''' + +ENV_2 = ''' + parameters: + p2: CORRECT + resource_registry: + test::R2: OS::Heat::RandomString + resources: + r3b: + test::R3: OS::Heat::RandomString +''' + + +class EnvironmentMergingTests(functional_base.FunctionalTestsBase): + + def test_server_environment_merging(self): + + # Setup + files = {'env1.yaml': ENV_1, 'env2.yaml': ENV_2} + environment_files = ['env1.yaml', 'env2.yaml'] + + # Test + stack_id = self.stack_create(stack_name='env_merge', + template=TEMPLATE, + files=files, + environment_files=environment_files) + + # Verify + + # Since there is no environment show, the registry overriding + # is partially verified by there being no error. If it wasn't + # working, test::R2 would remain mapped to BROKEN in env1. + + # Sanity check + resources = self.list_resources(stack_id) + self.assertEqual(4, len(resources)) + + # Verify the parameters are correctly set + stack = self.client.stacks.get(stack_id) + self.assertEqual('CORRECT', stack.parameters['p0']) + self.assertEqual('CORRECT', stack.parameters['p1']) + self.assertEqual('CORRECT', stack.parameters['p2']) + + # Verify that r3b has been overridden into a RandomString + # by checking to see that it has a value + r3b = self.client.resources.get(stack_id, 'r3b') + r3b_attrs = r3b.attributes + self.assertTrue('value' in r3b_attrs)