openstack/heat
Code Issues Proposed changes

Hook into environment merging on server

Browse Source 
If environment_files is specified, look in the files dict for each
specified environment file and merge into the stack's environment. This
is the same workflow that previously occurred client-side.

Change-Id: Ibe46fd35de7988920c101a83259c06c8f8a3ed0b
Implements: blueprint multi-environments
This commit is contained in:
Jay Dobies 2015-11-04 10:49:08 -05:00
parent 19e9ace7c1
commit d454caf00a
7 changed files with 272 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
heat/engine/service.py
View File

@ -69,6 +69,8 @@ from heat.objects import watch_data
from heat.objects import watch_rule
from heat.rpc import api as rpc_api
from heat.rpc import worker_api as rpc_worker_api
from heatclient.common import environment_format
from heatclient.common import template_utils
cfg.CONF.import_opt('engine_life_check_timeout', 'heat.common.config')
cfg.CONF.import_opt('max_resources_per_stack', 'heat.common.config')
@ -621,7 +623,8 @@ class EngineService(service.Service):
raise exception.RequestLimitExceeded(message=message)
def _parse_template_and_validate_stack(self, cnxt, stack_name, template,
params, files, args, owner_id=None,
params, files, environment_files,
args, owner_id=None,
nested_depth=0, user_creds_id=None,
stack_user_project_id=None,
convergence=False,
@ -641,6 +644,7 @@ class EngineService(service.Service):
new_params.update(params.get(rpc_api.STACK_PARAMETERS, {}))
params[rpc_api.STACK_PARAMETERS] = new_params
self._merge_environments(environment_files, files, params)
env = environment.Environment(params)
tmpl = templatem.Template(template, files=files, env=env)
@ -662,6 +666,30 @@ class EngineService(service.Service):
env.registry.log_resource_info(prefix=stack_name)
return stack
@staticmethod
def _merge_environments(environment_files, files, params):
"""Merges environment files into the stack input parameters.
If a list of environment files have been specified, this call will
pull the contents of each from the files dict, parse them as
environments, and merge them into the stack input params. This
behavior is the same as earlier versions of the Heat client that
performed this params population client-side.
:param environment_files: ordered names of the environment files
found in the files dict
:type environment_files: list or None
:param files: mapping of stack filenames to contents
:type files: dict
:param params: parameters describing the stack
:type dict:
"""
if environment_files:
for filename in environment_files:
raw_env = files[filename]
parsed_env = environment_format.parse(raw_env)
template_utils.deep_update(params, parsed_env)
@context.request_context
def preview_stack(self, cnxt, stack_name, template, params, files,
args, environment_files=None):
@ -689,6 +717,7 @@ class EngineService(service.Service):
template,
params,
files,
environment_files,
args,
convergence=conv_eng)
@ -751,9 +780,9 @@ class EngineService(service.Service):
convergence = cfg.CONF.convergence_engine
stack = self._parse_template_and_validate_stack(
cnxt, stack_name, template, params, files, args, owner_id,
nested_depth, user_creds_id, stack_user_project_id, convergence,
parent_resource_name)
cnxt, stack_name, template, params, files, environment_files,
args, owner_id, nested_depth, user_creds_id,
stack_user_project_id, convergence, parent_resource_name)
self.resource_enforcer.enforce_stack(stack)
stack_id = stack.store()
@ -786,6 +815,7 @@ class EngineService(service.Service):
:param files: Files referenced from the template
:param args: Request parameters/args passed from API
"""
# Now parse the template and any parameters for the updated
# stack definition. If PARAM_EXISTING is specified, we merge
# any environment provided into the existing one and attempt
@ -877,6 +907,9 @@ class EngineService(service.Service):
names included in the files dict
:type environment_files: list or None
"""
# Handle server-side environment file resolution
self._merge_environments(environment_files, files, params)
# Get the database representation of the existing stack
db_stack = self._get_stack(cnxt, stack_identity)
LOG.info(_LI('Updating stack %s'), db_stack.name)
@ -930,6 +963,9 @@ class EngineService(service.Service):
Note that at this stage the template has already been fetched from the
heat-api process if using a template-url.
"""
# Handle server-side environment file resolution
self._merge_environments(environment_files, files, params)
# Get the database representation of the existing stack
db_stack = self._get_stack(cnxt, stack_identity)
LOG.info(_LI('Previewing update of stack %s'), db_stack.name)
@ -1104,6 +1140,7 @@ class EngineService(service.Service):
service_check_defer = True
self._merge_environments(environment_files, files, params)
env = environment.Environment(params)
tmpl = templatem.Template(template, files=files, env=env)
try:

47
heat/tests/engine/service/test_service_engine.py
View File

@ -411,3 +411,50 @@ class ServiceEngineTest(common.HeatTestCase):
self.eng.start()
self.assertEqual(cfg.CONF.executor_thread_pool_size,
cfg.CONF.database.max_overflow)
def test_merge_environments(self):
# Setup
params = {'parameters': {
'p0': 'CORRECT',
'p1': 'INCORRECT',
'p2': 'INCORRECT'}
}
env_1 = '''
{'parameters' : {
'p1': 'CORRECT',
'p2': 'INCORRECT-ENV-1',
}}'''
env_2 = '''
{'parameters': {
'p2': 'CORRECT'
}}'''
files = {'env_1': env_1, 'env_2': env_2}
environment_files = ['env_1', 'env_2']
# Test
self.eng._merge_environments(environment_files, files, params)
# Verify
expected = {'parameters': {
'p0': 'CORRECT',
'p1': 'CORRECT',
'p2': 'CORRECT',
}}
self.assertEqual(expected, params)
def test_merge_environments_no_env_files(self):
params = {'parameters': {'p0': 'CORRECT'}}
env_1 = '''
{'parameters' : {
'p0': 'INCORRECT',
}}'''
files = {'env_1': env_1}
# Test - Should ignore env_1 in files
self.eng._merge_environments(None, files, params)
# Verify
expected = {'parameters': {'p0': 'CORRECT'}}
self.assertEqual(expected, params)

16
heat/tests/engine/service/test_stack_create.py
View File

@ -41,7 +41,8 @@ class StackCreateTest(common.HeatTestCase):
@mock.patch.object(threadgroup, 'ThreadGroup')
@mock.patch.object(stack.Stack, 'validate')
def _test_stack_create(self, stack_name, mock_validate, mock_tg):
def _test_stack_create(self, stack_name, mock_validate, mock_tg,
environment_files=None):
mock_tg.return_value = tools.DummyThreadGroup()
params = {'foo': 'bar'}
@ -53,8 +54,10 @@ class StackCreateTest(common.HeatTestCase):
mock_env = self.patchobject(environment, 'Environment',
return_value=stk.env)
mock_stack = self.patchobject(stack, 'Stack', return_value=stk)
mock_merge = self.patchobject(self.man, '_merge_environments')
result = self.man.create_stack(self.ctx, stack_name,
template, params, None, {})
template, params, None, {},
environment_files=environment_files)
self.assertEqual(stk.identifier(), result)
self.assertIsInstance(result, dict)
self.assertTrue(result['stack_id'])
@ -67,12 +70,21 @@ class StackCreateTest(common.HeatTestCase):
stack_user_project_id=None,
convergence=False,
parent_resource=None)
if environment_files:
mock_merge.assert_called_once_with(environment_files, None, params)
mock_validate.assert_called_once_with()
def test_stack_create(self):
stack_name = 'service_create_test_stack'
self._test_stack_create(stack_name)
def test_stack_create_with_environment_files(self):
stack_name = 'env_files_test_stack'
environment_files = ['env_1', 'env_2']
self._test_stack_create(stack_name,
environment_files=environment_files)
def test_stack_create_equals_max_per_tenant(self):
cfg.CONF.set_override('max_stacks_per_tenant', 1)
stack_name = 'service_create_test_stack_equals_max'

59
heat/tests/engine/service/test_stack_update.py
View File

@ -96,6 +96,37 @@ class ServiceStackUpdateTest(common.HeatTestCase):
mock_load.assert_called_once_with(self.ctx, stack=s)
mock_validate.assert_called_once_with()
def test_stack_update_with_environment_files(self):
# Setup
stack_name = 'service_update_env_files_stack'
params = {}
template = '{ "Template": "data" }'
old_stack = tools.get_stack(stack_name, self.ctx)
sid = old_stack.store()
old_stack.set_stack_user_project_id('1234')
stack_object.Stack.get_by_id(self.ctx, sid)
stk = tools.get_stack(stack_name, self.ctx)
# prepare mocks
self.patchobject(stack, 'Stack', return_value=stk)
self.patchobject(stack.Stack, 'load', return_value=old_stack)
self.patchobject(templatem, 'Template', return_value=stk.t)
self.patchobject(environment, 'Environment', return_value=stk.env)
self.patchobject(stk, 'validate', return_value=None)
self.patchobject(grevent, 'Event', return_value=mock.Mock())
mock_merge = self.patchobject(self.man, '_merge_environments')
# Test
environment_files = ['env_1']
self.man.update_stack(self.ctx, old_stack.identifier(),
template, params, None, {},
environment_files=environment_files)
# Verify
mock_merge.assert_called_once_with(environment_files, None, params)
def test_stack_update_existing_parameters(self):
# Use a template with existing parameters, then update the stack
# with a template containing additional parameters and ensure all
@ -712,7 +743,8 @@ resources:
self.man = service.EngineService('a-host', 'a-topic')
self.man.thread_group_mgr = tools.DummyThreadGroupManager()
def _test_stack_update_preview(self, orig_template, new_template):
def _test_stack_update_preview(self, orig_template, new_template,
environment_files=None):
stack_name = 'service_update_test_stack_preview'
params = {'foo': 'bar'}
old_stack = tools.get_stack(stack_name, self.ctx,
@ -731,6 +763,7 @@ resources:
mock_env = self.patchobject(environment, 'Environment',
return_value=stk.env)
mock_validate = self.patchobject(stk, 'validate', return_value=None)
mock_merge = self.patchobject(self.man, '_merge_environments')
# Patch _resolve_all_attributes or it tries to call novaclient
self.patchobject(resource.Resource, '_resolve_all_attributes',
@ -738,10 +771,13 @@ resources:
# do preview_update_stack
api_args = {'timeout_mins': 60}
result = self.man.preview_update_stack(self.ctx,
old_stack.identifier(),
new_template, params, None,
api_args)
result = self.man.preview_update_stack(
self.ctx,
old_stack.identifier(),
new_template, params, None,
api_args,
environment_files=environment_files)
# assertions
mock_stack.assert_called_once_with(
self.ctx, stk.name, stk.t, convergence=False,
@ -757,6 +793,9 @@ resources:
mock_env.assert_called_once_with(params)
mock_validate.assert_called_once_with()
if environment_files:
mock_merge.assert_called_once_with(environment_files, None, params)
return result
def test_stack_update_preview_added_unchanged(self):
@ -824,3 +863,13 @@ resources:
for section in empty_sections:
section_contents = [x for x in result[section]]
self.assertEqual([], section_contents)
def test_stack_update_preview_with_environment_files(self):
# Setup
environment_files = ['env_1']
# Test
self._test_stack_update_preview(self.old_tmpl, self.new_tmpl,
environment_files=environment_files)
# Assertions done in _test_stack_update_preview

22
heat/tests/test_engine_service.py
View File

@ -1123,7 +1123,7 @@ class StackServiceTest(common.HeatTestCase):
self.assertEqual(0, len(sl))
def _preview_stack(self):
def _preview_stack(self, environment_files=None):
res._register_class('GenericResource1', generic_rsrc.GenericResource)
res._register_class('GenericResource2', generic_rsrc.GenericResource)
@ -1138,7 +1138,8 @@ class StackServiceTest(common.HeatTestCase):
'SampleResource2': {'Type': 'GenericResource2'}}}
return self.eng.preview_stack(self.ctx, stack_name, tpl,
params, files, args)
params, files, args,
environment_files=environment_files)
def test_preview_stack_returns_a_stack(self):
stack = self._preview_stack()
@ -1181,6 +1182,17 @@ class StackServiceTest(common.HeatTestCase):
self._preview_stack)
self.assertEqual(exception.StackValidationFailed, ex.exc_info[0])
@mock.patch.object(service.EngineService, '_merge_environments')
def test_preview_environment_files(self, mock_merge):
# Setup
environment_files = ['env_1']
# Test
self._preview_stack(environment_files=environment_files)
# Verify
mock_merge.assert_called_once_with(environment_files, None, {})
@mock.patch.object(stack_object.Stack, 'get_by_name')
def test_validate_new_stack_checks_existing_stack(self, mock_stack_get):
mock_stack_get.return_value = 'existing_db_stack'
@ -1389,13 +1401,13 @@ class StackServiceTest(common.HeatTestCase):
# get parameters from adopt stack data which doesn't have it.
args = {"adopt_stack_data": '''{}'''}
self.eng._parse_template_and_validate_stack(
self.ctx, 'stack_name', template, {}, {}, args)
self.ctx, 'stack_name', template, {}, {}, None, args)
args = {"adopt_stack_data": '''{
"environment": {}
}'''}
self.eng._parse_template_and_validate_stack(
self.ctx, 'stack_name', template, {}, {}, args)
self.ctx, 'stack_name', template, {}, {}, None, args)
def test_parse_adopt_stack_data_with_parameters(self):
cfg.CONF.set_override('enable_stack_adopt', True)
@ -1420,5 +1432,5 @@ class StackServiceTest(common.HeatTestCase):
}
}}'''}
stack = self.eng._parse_template_and_validate_stack(
self.ctx, 'stack_name', template, {}, {}, args)
self.ctx, 'stack_name', template, {}, {}, None, args)
self.assertEqual(1, stack.parameters['volsize'])

6
heat_integrationtests/common/test.py
View File

@ -475,7 +475,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
def stack_create(self, stack_name=None, template=None, files=None,
parameters=None, environment=None, tags=None,
expected_status='CREATE_COMPLETE',
disable_rollback=True, enable_cleanup=True):
disable_rollback=True, enable_cleanup=True,
environment_files=None):
name = stack_name or self._stack_rand_name()
templ = template or self.template
templ_files = files or {}
@ -488,7 +489,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
disable_rollback=disable_rollback,
parameters=params,
environment=env,
tags=tags
tags=tags,
environment_files=environment_files
)
if expected_status not in ['ROLLBACK_COMPLETE'] and enable_cleanup:
self.addCleanup(self._stack_delete, name)

95
heat_integrationtests/functional/test_env_merge.py Normal file
View File

@ -0,0 +1,95 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from heat_integrationtests.functional import functional_base
TEMPLATE = '''
heat_template_version: 2015-04-30
parameters:
p0:
type: string
default: CORRECT
p1:
type: string
default: INCORRECT
p2:
type: string
default: INCORRECT
resources:
r1:
type: test::R1
r2:
type: test::R2
r3a:
type: test::R3
r3b:
type: test::R3
'''
ENV_1 = '''
parameters:
p1: CORRECT
p2: INCORRECT-E1
resource_registry:
test::R1: OS::Heat::RandomString
test::R2: BROKEN
test::R3: OS::Heat::None
'''
ENV_2 = '''
parameters:
p2: CORRECT
resource_registry:
test::R2: OS::Heat::RandomString
resources:
r3b:
test::R3: OS::Heat::RandomString
'''
class EnvironmentMergingTests(functional_base.FunctionalTestsBase):
def test_server_environment_merging(self):
# Setup
files = {'env1.yaml': ENV_1, 'env2.yaml': ENV_2}
environment_files = ['env1.yaml', 'env2.yaml']
# Test
stack_id = self.stack_create(stack_name='env_merge',
template=TEMPLATE,
files=files,
environment_files=environment_files)
# Verify
# Since there is no environment show, the registry overriding
# is partially verified by there being no error. If it wasn't
# working, test::R2 would remain mapped to BROKEN in env1.
# Sanity check
resources = self.list_resources(stack_id)
self.assertEqual(4, len(resources))
# Verify the parameters are correctly set
stack = self.client.stacks.get(stack_id)
self.assertEqual('CORRECT', stack.parameters['p0'])
self.assertEqual('CORRECT', stack.parameters['p1'])
self.assertEqual('CORRECT', stack.parameters['p2'])
# Verify that r3b has been overridden into a RandomString
# by checking to see that it has a value
r3b = self.client.resources.get(stack_id, 'r3b')
r3b_attrs = r3b.attributes
self.assertTrue('value' in r3b_attrs)