From 8efa2649bce040de1f10ecfe4ba01899039a7837 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Mon, 2 Dec 2013 14:47:23 +1300 Subject: [PATCH] Allow intrinsic functions to be called in any order An optional transform function can be passed into the resolve functions, which allows transforms to occur on the child snippets. Change-Id: I5d9e061c6a55f3939c5e03c780eaad72a001fccc Closes-Bug: #1256742 --- heat/engine/hot.py | 19 ++++--- heat/engine/parser.py | 5 +- heat/engine/template.py | 72 ++++++++++++++---------- heat/tests/test_parser.py | 114 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 166 insertions(+), 44 deletions(-) diff --git a/heat/engine/hot.py b/heat/engine/hot.py index 3599fa0e5e..ff6a8c328d 100644 --- a/heat/engine/hot.py +++ b/heat/engine/hot.py @@ -148,7 +148,7 @@ class HOTemplate(template.Template): return cfn_outputs @staticmethod - def resolve_param_refs(s, parameters): + def resolve_param_refs(s, parameters, transform=None): """ Resolve constructs of the form { get_param: my_param } """ @@ -163,10 +163,11 @@ class HOTemplate(template.Template): except (KeyError, ValueError): raise exception.UserParameterMissing(key=ref) - return template._resolve(match_param_ref, handle_param_ref, s) + return template._resolve(match_param_ref, handle_param_ref, s, + transform) @staticmethod - def resolve_resource_refs(s, resources): + def resolve_resource_refs(s, resources, transform=None): ''' Resolve constructs of the form { "get_resource" : "resource" } ''' @@ -176,10 +177,11 @@ class HOTemplate(template.Template): def handle_resource_ref(arg): return resources[arg].FnGetRefId() - return template._resolve(match_resource_ref, handle_resource_ref, s) + return template._resolve(match_resource_ref, handle_resource_ref, s, + transform) @staticmethod - def resolve_attributes(s, resources): + def resolve_attributes(s, resources, transform=None): """ Resolve constructs of the form { get_attr: [my_resource, my_attr] } """ @@ -206,10 +208,11 @@ class HOTemplate(template.Template): raise exception.InvalidTemplateAttribute(resource=resource, key=att) - return template._resolve(match_get_attr, handle_get_attr, s) + return template._resolve(match_get_attr, handle_get_attr, s, + transform) @staticmethod - def resolve_replace(s): + def resolve_replace(s, transform=None): """ Resolve template string substitution via function str_replace @@ -253,7 +256,7 @@ class HOTemplate(template.Template): match_str_replace = lambda k, v: k in ['str_replace', 'Fn::Replace'] return template._resolve(match_str_replace, - handle_str_replace, s) + handle_str_replace, s, transform) def param_schemata(self): params = self[PARAMETERS].iteritems() diff --git a/heat/engine/parser.py b/heat/engine/parser.py index ffe23764c9..56d3159360 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -728,6 +728,9 @@ def transform(data, transformations): Apply each of the transformation functions in the supplied list to the data in turn. ''' + def sub_transform(d): + return transform(d, transformations) + for t in transformations: - data = t(data) + data = t(data, transform=sub_transform) return data diff --git a/heat/engine/template.py b/heat/engine/template.py index 69f838b872..9dffbdd405 100644 --- a/heat/engine/template.py +++ b/heat/engine/template.py @@ -87,7 +87,7 @@ class Template(collections.Mapping): '''Return the number of sections.''' return len(SECTIONS) - def resolve_find_in_map(self, s): + def resolve_find_in_map(self, s, transform=None): ''' Resolve constructs of the form { "Fn::FindInMap" : [ "mapping", "key", @@ -101,10 +101,10 @@ class Template(collections.Mapping): raise KeyError(str(ex)) return _resolve(lambda k, v: k == 'Fn::FindInMap', - handle_find_in_map, s) + handle_find_in_map, s, transform) @staticmethod - def resolve_availability_zones(s, stack): + def resolve_availability_zones(s, stack, transform=None): ''' looking for { "Fn::GetAZs" : "str" } ''' @@ -118,10 +118,10 @@ class Template(collections.Mapping): else: return stack.get_availability_zones() - return _resolve(match_get_az, handle_get_az, s) + return _resolve(match_get_az, handle_get_az, s, transform) @staticmethod - def resolve_param_refs(s, parameters): + def resolve_param_refs(s, parameters, transform=None): ''' Resolve constructs of the form { "Ref" : "string" } ''' @@ -136,10 +136,10 @@ class Template(collections.Mapping): except (KeyError, ValueError): raise exception.UserParameterMissing(key=ref) - return _resolve(match_param_ref, handle_param_ref, s) + return _resolve(match_param_ref, handle_param_ref, s, transform) @staticmethod - def resolve_resource_refs(s, resources): + def resolve_resource_refs(s, resources, transform=None): ''' Resolve constructs of the form { "Ref" : "resource" } ''' @@ -149,10 +149,10 @@ class Template(collections.Mapping): def handle_resource_ref(arg): return resources[arg].FnGetRefId() - return _resolve(match_resource_ref, handle_resource_ref, s) + return _resolve(match_resource_ref, handle_resource_ref, s, transform) @staticmethod - def resolve_attributes(s, resources): + def resolve_attributes(s, resources, transform=None): ''' Resolve constructs of the form { "Fn::GetAtt" : [ "WebServer", "PublicIp" ] } @@ -173,10 +173,11 @@ class Template(collections.Mapping): raise exception.InvalidTemplateAttribute(resource=resource, key=att) - return _resolve(lambda k, v: k == 'Fn::GetAtt', handle_getatt, s) + return _resolve(lambda k, v: k == 'Fn::GetAtt', handle_getatt, s, + transform) @staticmethod - def reduce_joins(s): + def reduce_joins(s, transform=None): ''' Reduces contiguous strings in Fn::Join to a single joined string eg the following @@ -212,10 +213,11 @@ class Template(collections.Mapping): reduced.append(delim.join(contiguous)) return {'Fn::Join': [delim, reduced]} - return _resolve(lambda k, v: k == 'Fn::Join', handle_join, s) + return _resolve(lambda k, v: k == 'Fn::Join', handle_join, s, + transform) @staticmethod - def resolve_select(s): + def resolve_select(s, transform=None): ''' Resolve constructs of the form: (for a list lookup) @@ -278,10 +280,11 @@ class Template(collections.Mapping): raise TypeError(_('Arguments to "Fn::Select" not fully resolved')) - return _resolve(lambda k, v: k == 'Fn::Select', handle_select, s) + return _resolve(lambda k, v: k == 'Fn::Select', handle_select, s, + transform) @staticmethod - def resolve_joins(s): + def resolve_joins(s, transform=None): ''' Resolve constructs of the form { "Fn::Join" : [ "delim", [ "str1", "str2" ] } @@ -310,10 +313,11 @@ class Template(collections.Mapping): return delim.join(empty_for_none(value) for value in strings) - return _resolve(lambda k, v: k == 'Fn::Join', handle_join, s) + return _resolve(lambda k, v: k == 'Fn::Join', handle_join, s, + transform) @staticmethod - def resolve_split(s): + def resolve_split(s, transform=None): ''' Split strings in Fn::Split to a list of sub strings eg the following @@ -336,10 +340,11 @@ class Template(collections.Mapping): '"Fn::Split" should be: %s') % example) return strings.split(delim) - return _resolve(lambda k, v: k == 'Fn::Split', handle_split, s) + return _resolve(lambda k, v: k == 'Fn::Split', handle_split, s, + transform) @staticmethod - def resolve_replace(s): + def resolve_replace(s, transform=None): """ Resolve constructs of the form:: @@ -384,10 +389,11 @@ class Template(collections.Mapping): string = string.replace(k, v) return string - return _resolve(lambda k, v: k == 'Fn::Replace', handle_replace, s) + return _resolve(lambda k, v: k == 'Fn::Replace', handle_replace, s, + transform) @staticmethod - def resolve_base64(s): + def resolve_base64(s, transform=None): ''' Resolve constructs of the form { "Fn::Base64" : "string" } ''' @@ -397,10 +403,11 @@ class Template(collections.Mapping): 'not fully resolved')) return string - return _resolve(lambda k, v: k == 'Fn::Base64', handle_base64, s) + return _resolve(lambda k, v: k == 'Fn::Base64', handle_base64, s, + transform) @staticmethod - def resolve_member_list_to_map(s): + def resolve_member_list_to_map(s, transform=None): ''' Resolve constructs of the form:: @@ -437,10 +444,10 @@ class Template(collections.Mapping): valuename=args[1]) return _resolve(lambda k, v: k == 'Fn::MemberListToMap', - handle_member_list_to_map, s) + handle_member_list_to_map, s, transform) @staticmethod - def resolve_resource_facade(s, stack): + def resolve_resource_facade(s, stack, transform=None): ''' Resolve constructs of the form {'Fn::ResourceFacade': 'Metadata'} ''' @@ -462,29 +469,34 @@ class Template(collections.Mapping): return _resolve(lambda k, v: k == 'Fn::ResourceFacade', handle_resource_facade, - s) + s, transform) def param_schemata(self): parameters = self[PARAMETERS].iteritems() return dict((name, ParamSchema(schema)) for name, schema in parameters) -def _resolve(match, handle, snippet): +def _resolve(match, handle, snippet, transform=None): ''' Resolve constructs in a snippet of a template. The supplied match function should return True if a particular key-value pair should be substituted, and the handle function should return the correct substitution when passed the argument list as parameters. - Returns a copy of the original snippet with the substitutions performed. + Returns a copy of the original snippet with the substitutions to handle + functions performed. ''' - recurse = lambda s: _resolve(match, handle, s) + + recurse = lambda s: _resolve(match, handle, s, transform) if isinstance(snippet, dict): if len(snippet) == 1: k, v = snippet.items()[0] if match(k, v): - return handle(recurse(v)) + if transform: + return handle(transform(v)) + else: + return handle(v) return dict((k, recurse(v)) for k, v in snippet.items()) elif isinstance(snippet, list): return [recurse(s) for s in snippet] diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py index 016cca0fad..7fdc5c7c55 100644 --- a/heat/tests/test_parser.py +++ b/heat/tests/test_parser.py @@ -106,11 +106,6 @@ class ParserTest(HeatTestCase): self.assertEqual(parsed['blarg'], raw['blarg']) self.assertTrue(parsed is not raw) - def test_join_recursive(self): - raw = {'Fn::Join': ['\n', [{'Fn::Join': - [' ', ['foo', 'bar']]}, 'baz']]} - self.assertEqual(join(raw), 'foo bar\nbaz') - mapping_template = template_format.parse('''{ "Mappings" : { @@ -608,6 +603,115 @@ class TemplateFnErrorTest(HeatTestCase): self.assertIn(self.snippet.keys()[0], str(error)) +class ResolveDataTest(HeatTestCase): + + def setUp(self): + super(ResolveDataTest, self).setUp() + self.username = 'parser_stack_test_user' + + utils.setup_dummy_db() + self.ctx = utils.dummy_context() + + self.stack = parser.Stack(self.ctx, 'resolve_test_stack', + template.Template({}), + environment.Environment({})) + + def test_split_join_split_join(self): + # each snippet in this test encapsulates + # the snippet from the previous step, leading + # to increasingly nested function calls + + # split + snippet = {'Fn::Split': [',', 'one,two,three']} + self.assertEqual(['one', 'two', 'three'], + self.stack.resolve_runtime_data(snippet)) + + # split then join + snippet = {'Fn::Join': [';', snippet]} + self.assertEqual('one;two;three', + self.stack.resolve_runtime_data(snippet)) + + # split then join then split + snippet = {'Fn::Split': [';', snippet]} + self.assertEqual(['one', 'two', 'three'], + self.stack.resolve_runtime_data(snippet)) + + # split then join then split then join + snippet = {'Fn::Join': ['-', snippet]} + self.assertEqual('one-two-three', + self.stack.resolve_runtime_data(snippet)) + + def test_join_recursive(self): + raw = {'Fn::Join': ['\n', [{'Fn::Join': + [' ', ['foo', 'bar']]}, 'baz']]} + self.assertEqual('foo bar\nbaz', self.stack.resolve_runtime_data(raw)) + + def test_base64_replace(self): + raw = {'Fn::Base64': {'Fn::Replace': [ + {'foo': 'bar'}, 'Meet at the foo']}} + self.assertEqual('Meet at the bar', + self.stack.resolve_runtime_data(raw)) + + def test_replace_base64(self): + raw = {'Fn::Replace': [{'foo': 'bar'}, { + 'Fn::Base64': 'Meet at the foo'}]} + self.assertEqual('Meet at the bar', + self.stack.resolve_runtime_data(raw)) + + def test_nested_selects(self): + data = { + 'a': ['one', 'two', 'three'], + 'b': ['een', 'twee', {'d': 'D', 'e': 'E'}] + } + raw = {'Fn::Select': ['a', data]} + self.assertEqual(data['a'], + self.stack.resolve_runtime_data(raw)) + + raw = {'Fn::Select': ['b', data]} + self.assertEqual(data['b'], + self.stack.resolve_runtime_data(raw)) + + raw = { + 'Fn::Select': ['1', { + 'Fn::Select': ['b', data]}]} + self.assertEqual('twee', + self.stack.resolve_runtime_data(raw)) + + raw = { + 'Fn::Select': ['e', { + 'Fn::Select': ['2', { + 'Fn::Select': ['b', data]}]}]} + self.assertEqual('E', + self.stack.resolve_runtime_data(raw)) + + def test_member_list_select(self): + snippet = {'Fn::Select': ['metric', {"Fn::MemberListToMap": [ + 'Name', 'Value', ['.member.0.Name=metric', + '.member.0.Value=cpu', + '.member.1.Name=size', + '.member.1.Value=56']]}]} + self.assertEqual('cpu', + self.stack.resolve_runtime_data(snippet)) + + def test_join_reduce(self): + join = {"Fn::Join": [" ", ["foo", "bar", "baz", {'Ref': 'baz'}, + "bink", "bonk"]]} + self.assertEqual( + {"Fn::Join": [" ", ["foo bar baz", {'Ref': 'baz'}, "bink bonk"]]}, + self.stack.resolve_static_data(join)) + + join = {"Fn::Join": [" ", ["foo", {'Ref': 'baz'}, + "bink"]]} + self.assertEqual( + {"Fn::Join": [" ", ["foo", {'Ref': 'baz'}, "bink"]]}, + self.stack.resolve_static_data(join)) + + join = {"Fn::Join": [" ", [{'Ref': 'baz'}]]} + self.assertEqual( + {"Fn::Join": [" ", [{'Ref': 'baz'}]]}, + self.stack.resolve_static_data(join)) + + class StackTest(HeatTestCase): def setUp(self): super(StackTest, self).setUp()