Improve validation during template.parse

Now error with a path to the problem part of template will be
raised while parsing template. It affects both resource
definitions and outputs parsing. Note, that during parsing of
template we only instantiate the function objects, but doesn't
make any function validation. The function validation still
doesn't return any path with error. This will be fixed in the
other patch.

Change-Id: Ic5b960e79a54e88087803bc092f614911d7e995a
This commit is contained in:
Oleksii Chuprykov 2016-06-17 20:12:02 +03:00
parent 6499a52e4c
commit d6e36c2dd2
7 changed files with 129 additions and 87 deletions

View File

@ -104,7 +104,8 @@ class CfnTemplate(template.Template):
try:
for name, snippet in resources.items():
data = self.parse(stack, snippet)
path = '.'.join([self.RESOURCES, name])
data = self.parse(stack, snippet, path)
if not self.validate_resource_key_type(RES_TYPE,
six.string_types,

View File

@ -209,7 +209,8 @@ class HOTemplate20130523(template.Template):
try:
for name, snippet in resources.items():
data = self.parse(stack, snippet)
path = '.'.join([self.RESOURCES, name])
data = self.parse(stack, snippet, path)
if not self.validate_resource_key_type(RES_TYPE,
six.string_types,

View File

@ -228,7 +228,8 @@ class Stack(collections.Mapping):
self._set_param_stackid()
if resolve_data:
self.outputs = self.resolve_static_data(self.t[self.t.OUTPUTS])
self.outputs = self.resolve_static_data(
self.t[self.t.OUTPUTS], path=self.t.OUTPUTS)
else:
self.outputs = {}
@ -1450,7 +1451,8 @@ class Stack(collections.Mapping):
previous_template_id = self.t.id
self.t = newstack.t
template_outputs = self.t[self.t.OUTPUTS]
self.outputs = self.resolve_static_data(template_outputs)
self.outputs = self.resolve_static_data(
template_outputs, path=self.t.OUTPUTS)
finally:
if should_rollback:
# Already handled in rollback task
@ -1908,14 +1910,8 @@ class Stack(collections.Mapping):
'tags': self.tags,
}
def resolve_static_data(self, snippet):
try:
return self.t.parse(self, snippet)
except AssertionError:
raise
except Exception as ex:
raise exception.StackValidationFailed(
message=encodeutils.safe_decode(six.text_type(ex)))
def resolve_static_data(self, snippet, path=''):
return self.t.parse(self, snippet, path=path)
def reset_resource_attributes(self):
# nothing is cached if no resources exist

View File

@ -230,8 +230,8 @@ class Template(collections.Mapping):
if self.RESOURCES in self.t:
self.t.update({self.RESOURCES: {}})
def parse(self, stack, snippet):
return parse(self.functions, stack, snippet)
def parse(self, stack, snippet, path=''):
return parse(self.functions, stack, snippet, path)
def validate(self):
"""Validate the template.
@ -299,18 +299,33 @@ class Template(collections.Mapping):
return cls(tmpl)
def parse(functions, stack, snippet):
def parse(functions, stack, snippet, path=''):
recurse = functools.partial(parse, functions, stack)
if isinstance(snippet, collections.Mapping):
def mkpath(key):
return '.'.join([path, six.text_type(key)])
if len(snippet) == 1:
fn_name, args = next(six.iteritems(snippet))
Func = functions.get(fn_name)
if Func is not None:
return Func(stack, fn_name, recurse(args))
return dict((k, recurse(v)) for k, v in six.iteritems(snippet))
try:
path = '.'.join([path, fn_name])
return Func(stack, fn_name, recurse(args, path))
except (ValueError, TypeError, KeyError) as e:
raise exception.StackValidationFailed(
path=path,
message=six.text_type(e))
return dict((k, recurse(v, mkpath(k)))
for k, v in six.iteritems(snippet))
elif (not isinstance(snippet, six.string_types) and
isinstance(snippet, collections.Iterable)):
return [recurse(v) for v in snippet]
def mkpath(idx):
return ''.join([path, '[%d]' % idx])
return [recurse(v, mkpath(i)) for i, v in enumerate(snippet)]
else:
return snippet

View File

@ -627,7 +627,8 @@ class HOTemplateTest(common.HeatTestCase):
tmpl = template.Template(hot_tpl_empty)
self.assertRaises(TypeError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
def test_str_replace_invalid_param_keys(self):
"""Test str_replace function parameter keys.
@ -641,12 +642,14 @@ class HOTemplateTest(common.HeatTestCase):
tmpl = template.Template(hot_tpl_empty)
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
snippet = {'str_replace': {'tmpl': 'Template var1 string var2',
'parms': {'var1': 'foo', 'var2': 'bar'}}}
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
def test_str_replace_invalid_param_types(self):
"""Test str_replace function parameter values.
@ -665,8 +668,10 @@ class HOTemplateTest(common.HeatTestCase):
snippet = {'str_replace': {'template': 'Template var1 string var2',
'params': ['var1', 'foo', 'var2', 'bar']}}
ex = self.assertRaises(TypeError, self.resolve, snippet, tmpl)
self.assertIn('parameters must be a mapping', six.text_type(ex))
ex = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
self.assertIn('.str_replace: "str_replace" parameters must be a'
' mapping', six.text_type(ex))
def test_str_replace_invalid_param_type_init(self):
"""Test str_replace function parameter values.
@ -824,32 +829,42 @@ class HOTemplateTest(common.HeatTestCase):
def test_join_invalid(self):
snippet = {'list_join': 'bad'}
l_tmpl = template.Template(hot_liberty_tpl_empty)
exc = self.assertRaises(TypeError, self.resolve, snippet, l_tmpl)
self.assertIn('Incorrect arguments', six.text_type(exc))
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, l_tmpl)
self.assertIn('.list_join: Incorrect arguments to "list_join"',
six.text_type(exc))
k_tmpl = template.Template(hot_kilo_tpl_empty)
exc1 = self.assertRaises(TypeError, self.resolve, snippet, k_tmpl)
self.assertIn('Incorrect arguments', six.text_type(exc1))
exc1 = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, k_tmpl)
self.assertIn('.list_join: Incorrect arguments to "list_join"',
six.text_type(exc1))
def test_join_int_invalid(self):
snippet = {'list_join': 5}
l_tmpl = template.Template(hot_liberty_tpl_empty)
exc = self.assertRaises(TypeError, self.resolve, snippet, l_tmpl)
self.assertIn('Incorrect arguments', six.text_type(exc))
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, l_tmpl)
self.assertIn('.list_join: Incorrect arguments', six.text_type(exc))
k_tmpl = template.Template(hot_kilo_tpl_empty)
exc1 = self.assertRaises(TypeError, self.resolve, snippet, k_tmpl)
self.assertIn('Incorrect arguments', six.text_type(exc1))
exc1 = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, k_tmpl)
self.assertIn('.list_join: Incorrect arguments', six.text_type(exc1))
def test_join_invalid_value(self):
snippet = {'list_join': [',']}
l_tmpl = template.Template(hot_liberty_tpl_empty)
exc = self.assertRaises(ValueError, self.resolve, snippet, l_tmpl)
self.assertIn('Incorrect arguments', six.text_type(exc))
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, l_tmpl)
self.assertIn('.list_join: Incorrect arguments to "list_join"',
six.text_type(exc))
k_tmpl = template.Template(hot_kilo_tpl_empty)
exc1 = self.assertRaises(ValueError, self.resolve, snippet, k_tmpl)
self.assertIn('Incorrect arguments', six.text_type(exc1))
exc1 = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, k_tmpl)
self.assertIn('.list_join: Incorrect arguments to "list_join"',
six.text_type(exc1))
def test_join_invalid_multiple(self):
snippet = {'list_join': [',', 'bad', ['foo']]}
@ -907,19 +922,22 @@ class HOTemplateTest(common.HeatTestCase):
'data': 'mustbeamap',
'bogus': ""}}
tmpl = template.Template(hot_newton_tpl_empty)
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
def test_yaql_invalid_syntax(self):
snippet = {'yaql': {'wrong': 'wrong_expr',
'wrong_data': 'mustbeamap'}}
tmpl = template.Template(hot_newton_tpl_empty)
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
def test_yaql_non_map_args(self):
snippet = {'yaql': 'invalid'}
tmpl = template.Template(hot_newton_tpl_empty)
msg = 'Arguments to "yaql" must be a map.'
self.assertRaisesRegexp(TypeError, msg, self.resolve, snippet, tmpl)
msg = '.yaql: Arguments to "yaql" must be a map.'
self.assertRaisesRegexp(exception.StackValidationFailed,
msg, self.resolve, snippet, tmpl)
def test_yaql_invalid_expression(self):
snippet = {'yaql': {'expression': 'invalid(',
@ -968,13 +986,15 @@ class HOTemplateTest(common.HeatTestCase):
tmpl = template.Template(hot_newton_tpl_empty)
snippet = {'equals': ['test', 'prod', 'invalid']}
exc = self.assertRaises(ValueError, self.resolve, snippet, tmpl)
self.assertIn('Arguments to "equals" must be of the form: '
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
self.assertIn('.equals: Arguments to "equals" must be of the form: '
'[value_1, value_2]', six.text_type(exc))
snippet = {'equals': "invalid condition"}
exc = self.assertRaises(ValueError, self.resolve, snippet, tmpl)
self.assertIn('Arguments to "equals" must be of the form: '
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
self.assertIn('.equals: Arguments to "equals" must be of the form: '
'[value_1, value_2]', six.text_type(exc))
def test_repeat(self):
@ -1054,12 +1074,14 @@ class HOTemplateTest(common.HeatTestCase):
# missing for_each
snippet = {'repeat': {'template': 'this is %var%'}}
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
# misspelled for_each
snippet = {'repeat': {'template': 'this is %var%',
'foreach': {'%var%': ['a', 'b', 'c']}}}
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
# value given to for_each entry is not a list
snippet = {'repeat': {'template': 'this is %var%',
@ -1069,7 +1091,8 @@ class HOTemplateTest(common.HeatTestCase):
# misspelled template
snippet = {'repeat': {'templte': 'this is %var%',
'for_each': {'%var%': ['a', 'b', 'c']}}}
self.assertRaises(KeyError, self.resolve, snippet, tmpl)
self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
def test_repeat_bad_arg_type(self):
tmpl = template.Template(hot_kilo_tpl_empty)
@ -1285,7 +1308,7 @@ class HOTemplateTest(common.HeatTestCase):
snippet = {'resource_facade': 'wibble'}
stack = parser.Stack(utils.dummy_context(), 'test_stack',
template.Template(hot_tpl_empty))
error = self.assertRaises(ValueError,
error = self.assertRaises(exception.StackValidationFailed,
self.resolve,
snippet,
stack.t, stack)
@ -2470,8 +2493,9 @@ class TestGetAttAllAttributes(common.HeatTestCase):
('test_get_attr_all_attributes_str', dict(
hot_tpl=hot_tpl_generic_resource_all_attrs,
snippet={'Value': {'get_attr': 'resource1'}},
expected='Argument to "get_attr" must be a list',
raises=TypeError
expected='.Value.get_attr: Argument to "get_attr" must be a '
'list',
raises=exception.StackValidationFailed
)),
('test_get_attr_all_attributes_invalid_resource_list', dict(
hot_tpl=hot_tpl_generic_resource_all_attrs,
@ -2483,23 +2507,24 @@ class TestGetAttAllAttributes(common.HeatTestCase):
('test_get_attr_all_attributes_invalid_type', dict(
hot_tpl=hot_tpl_generic_resource_all_attrs,
snippet={'Value': {'get_attr': {'resource1': 'attr1'}}},
raises=TypeError,
expected='Argument to "get_attr" must be a list'
raises=exception.StackValidationFailed,
expected='.Value.get_attr: Argument to "get_attr" must be a '
'list'
)),
('test_get_attr_all_attributes_invalid_arg_str', dict(
hot_tpl=hot_tpl_generic_resource_all_attrs,
snippet={'Value': {'get_attr': ''}},
raises=ValueError,
expected='Arguments to "get_attr" can be of the next '
'forms: [resource_name] or '
raises=exception.StackValidationFailed,
expected='.Value.get_attr: Arguments to "get_attr" can be of '
'the next forms: [resource_name] or '
'[resource_name, attribute, (path), ...]'
)),
('test_get_attr_all_attributes_invalid_arg_list', dict(
hot_tpl=hot_tpl_generic_resource_all_attrs,
snippet={'Value': {'get_attr': []}},
raises=ValueError,
expected='Arguments to "get_attr" can be of the next '
'forms: [resource_name] or '
raises=exception.StackValidationFailed,
expected='.Value.get_attr: Arguments to "get_attr" can be of '
'the next forms: [resource_name] or '
'[resource_name, attribute, (path), ...]'
)),
('test_get_attr_all_attributes_standard', dict(

View File

@ -602,7 +602,8 @@ class TemplateTest(common.HeatTestCase):
{'Fn::FindInMap': ["ReallyShortList"]})
for find in finds:
self.assertRaises(KeyError, self.resolve, find, tmpl, stk)
self.assertRaises(exception.StackValidationFailed,
self.resolve, find, tmpl, stk)
def test_param_refs(self):
env = environment.Environment({'foo': 'bar', 'blarg': 'wibble'})
@ -730,14 +731,16 @@ class TemplateTest(common.HeatTestCase):
tmpl = template.Template(empty_template20161014)
snippet = {'Fn::Equals': ['test', 'prod', 'invalid']}
exc = self.assertRaises(ValueError, self.resolve, snippet, tmpl)
self.assertIn('Arguments to "Fn::Equals" must be of the form: '
'[value_1, value_2]', six.text_type(exc))
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
self.assertIn('.Fn::Equals: Arguments to "Fn::Equals" must be of '
'the form: [value_1, value_2]', six.text_type(exc))
# test invalid type
snippet = {'Fn::Equals': {"equal": False}}
exc = self.assertRaises(ValueError, self.resolve, snippet, tmpl)
self.assertIn('Arguments to "Fn::Equals" must be of the form: '
'[value_1, value_2]', six.text_type(exc))
exc = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, tmpl)
self.assertIn('.Fn::Equals: Arguments to "Fn::Equals" must be of '
'the form: [value_1, value_2]', six.text_type(exc))
def test_join(self):
tmpl = template.Template(empty_template)
@ -895,7 +898,7 @@ class TemplateTest(common.HeatTestCase):
snippet = {'Fn::ResourceFacade': 'wibble'}
stk = stack.Stack(self.ctx, 'test_stack',
template.Template(empty_template))
error = self.assertRaises(ValueError,
error = self.assertRaises(exception.StackValidationFailed,
self.resolve, snippet, stk.t, stk)
self.assertIn(next(iter(snippet)), six.text_type(error))
@ -1048,22 +1051,22 @@ class TemplateFnErrorTest(common.HeatTestCase):
dict(expect=ValueError,
snippet={"Fn::Select": ["not", "no json"]})),
('select_wrong_num_args_1',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Select": []})),
('select_wrong_num_args_2',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Select": ["4"]})),
('select_wrong_num_args_3',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Select": ["foo", {"foo": "bar"}, ""]})),
('select_wrong_num_args_4',
dict(expect=TypeError,
snippet={'Fn::Select': [['f'], {'f': 'food'}]})),
('split_no_delim',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Split": ["foo, bar, achoo"]})),
('split_no_list',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Split": "foo, bar, achoo"})),
('base64_list',
dict(expect=TypeError,
@ -1077,18 +1080,18 @@ class TemplateFnErrorTest(common.HeatTestCase):
{'$var1': 'foo', '%var2%': ['bar']},
'$var1 is %var2%']})),
('replace_list_mapping',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Replace": [
['var1', 'foo', 'var2', 'bar'],
'$var1 is ${var2}']})),
('replace_dict',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Replace": {}})),
('replace_missing_template',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Replace": [['var1', 'foo', 'var2', 'bar']]})),
('replace_none_template',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Replace": [['var2', 'bar'], None]})),
('replace_list_string',
dict(expect=TypeError,
@ -1102,46 +1105,46 @@ class TemplateFnErrorTest(common.HeatTestCase):
dict(expect=TypeError,
snippet={"Fn::Join": [" ", {"foo": "bar"}]})),
('join_wrong_num_args_1',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": []})),
('join_wrong_num_args_2',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": [" "]})),
('join_wrong_num_args_3',
dict(expect=ValueError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": [" ", {"foo": "bar"}, ""]})),
('join_string_nodelim',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": "o"})),
('join_string_nodelim_1',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": "oh"})),
('join_string_nodelim_2',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": "ohh"})),
('join_dict_nodelim1',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": {"foo": "bar"}})),
('join_dict_nodelim2',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": {"foo": "bar", "blarg": "wibble"}})),
('join_dict_nodelim3',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::Join": {"foo": "bar", "blarg": "wibble",
"baz": "quux"}})),
('member_list2map_no_key_or_val',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::MemberListToMap": [
'Key', ['.member.2.Key=metric',
'.member.2.Value=cpu',
'.member.5.Key=size',
'.member.5.Value=56']]})),
('member_list2map_no_list',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::MemberListToMap": [
'Key', '.member.2.Key=metric']})),
('member_list2map_not_string',
dict(expect=TypeError,
dict(expect=exception.StackValidationFailed,
snippet={"Fn::MemberListToMap": [
'Name', ['Value'], ['.member.0.Name=metric',
'.member.0.Value=cpu',

View File

@ -1624,7 +1624,8 @@ class ValidateTest(common.HeatTestCase):
template = tmpl.Template(t)
err = self.assertRaises(exception.StackValidationFailed,
parser.Stack, self.ctx, 'test_stack', template)
error_message = ('Arguments to "get_attr" must be of the form '
error_message = ('outputs.string.value.get_attr: Arguments to '
'"get_attr" must be of the form '
'[resource_name, attribute, (path), ...]')
self.assertEqual(error_message, six.text_type(err))