Refactor template resolution

Resolve functions in templates by making a copy of the data rather than
modifying the original. This means that e.g. a resource resolving functions
in its own template data does not result in changes to the data held by the
Stack.

This patch also refactors all of the template resolution methods to operate
using a common parsing algorithm to move through the tree.

Finally, the resources have been worked to load data as it is needed,
rather than requiring external code to put them into the correct state
before using them.

Change-Id: I79eafaefc9ced07b652fac7162aa2edbfa7f547a
Signed-off-by: Zane Bitter <zbitter@redhat.com>

heat/engine/eip.py

@@ -28,7 +28,18 @@ class ElasticIp(Resource):
 
     def __init__(self, name, json_snippet, stack):
         super(ElasticIp, self).__init__(name, json_snippet, stack)
-        self.ipaddress = ''
+        self.ipaddress = None
+
+    def _ipaddress(self):
+        if self.ipaddress is None:
+            if self.instance_id is not None:
+                try:
+                    ips = self.nova().floating_ips.get(self.instance_id)
+                except NotFound as ex:
+                    logger.warn("Floating IPs not found: %s" % str(ex))
+                else:
+                    self.ipaddress = ips.ip
+        return self.ipaddress or ''
 
     def create(self):
         """Allocate a floating IP for the current tenant."""
@@ -49,19 +60,6 @@ class ElasticIp(Resource):
         '''
         return Resource.validate(self)
 
-    def reload(self):
-        '''
-        get the ipaddress here
-        '''
-        if self.instance_id is not None:
-            try:
-                ips = self.nova().floating_ips.get(self.instance_id)
-                self.ipaddress = ips.ip
-            except Exception as ex:
-                logger.warn("Error getting floating IPs: %s" % str(ex))
-
-        Resource.reload(self)
-
     def delete(self):
         """De-allocate a floating IP."""
         if self.state == self.DELETE_IN_PROGRESS or \
@@ -77,7 +75,7 @@ class ElasticIp(Resource):
         self.state_set(self.DELETE_COMPLETE)
 
     def FnGetRefId(self):
-        return unicode(self.ipaddress)
+        return unicode(self._ipaddress())
 
     def FnGetAtt(self, key):
         if key == 'AllocationId':

heat/engine/instance.py

@@ -110,7 +110,7 @@ class Instance(Resource):
 
     def __init__(self, name, json_snippet, stack):
         super(Instance, self).__init__(name, json_snippet, stack)
-        self.ipaddress = '0.0.0.0'
+        self.ipaddress = None
         self.mime_string = None
 
         self.itype_oflavor = {'t1.micro': 'm1.tiny',
@@ -126,15 +126,30 @@ class Instance(Resource):
             'cc2.8xlarge': 'm1.large',
             'cg1.4xlarge': 'm1.large'}
 
-    def FnGetAtt(self, key):
+    def _ipaddress(self):
+        '''
+        Return the server's IP address, fetching it from Nova if necessary
+        '''
+        if self.ipaddress is None:
+            try:
+                server = self.nova().servers.get(self.instance_id)
+            except NotFound as ex:
+                logger.warn('Instance IP address not found (%s)' % str(ex))
+            else:
+                for n in server.networks:
+                    self.ipaddress = server.networks[n][0]
+                    break
 
+        return self.ipaddress or '0.0.0.0'
+
+    def FnGetAtt(self, key):
         res = None
         if key == 'AvailabilityZone':
             res = self.properties['AvailabilityZone']
         elif key == 'PublicIp':
-            res = self.ipaddress
+            res = self._ipaddress()
         elif key == 'PrivateDnsName':
-            res = self.ipaddress
+            res = self._ipaddress()
         else:
             raise exception.InvalidTemplateAttribute(resource=self.name,
                                                      key=key)
@@ -259,19 +274,6 @@ class Instance(Resource):
                         'Provided KeyName is not registered with nova'}
         return None
 
-    def reload(self):
-        '''
-        re-read the server's ipaddress so FnGetAtt works.
-        '''
-        try:
-            server = self.nova().servers.get(self.instance_id)
-            for n in server.networks:
-                self.ipaddress = server.networks[n][0]
-        except NotFound:
-            self.ipaddress = '0.0.0.0'
-
-        Resource.reload(self)
-
     def delete(self):
         if self.state == self.DELETE_IN_PROGRESS or \
            self.state == self.DELETE_COMPLETE:

heat/engine/parser.py

@@ -15,6 +15,7 @@
 
 import eventlet
 import json
+import itertools
 import logging
 from heat.common import exception
 from heat.engine import checkeddict
@@ -72,7 +73,7 @@ class Stack(object):
             res = Resource(rname, rdesc, self)
             self.resources[rname] = res
 
-            self.calulate_dependencies(rdesc, res)
+            self.calulate_dependencies(res.t, res)
 
     def validate(self):
         '''
@@ -233,23 +234,15 @@ class Stack(object):
         pool.spawn_n(self.delete_blocking)
 
     def get_outputs(self):
+        outputs = self.resolve_runtime_data(self.outputs)
 
-        for r in self.resources:
+        def output_dict(k):
-            self.resources[r].reload()
+            return {'Description': outputs[k].get('Description',
+                                                  'No description given'),
+                    'OutputKey': k,
+                    'OutputValue': outputs[k].get('Value', '')}
 
-        self.resolve_attributes(self.outputs)
+        return [output_dict(key) for key in outputs]
-        self.resolve_joins(self.outputs)
-
-        outs = []
-        for o in self.outputs:
-            out = {}
-            out['Description'] = self.outputs[o].get('Description',
-                                                     'No description given')
-            out['OutputKey'] = o
-            out['OutputValue'] = self.outputs[o].get('Value', '')
-            outs.append(out)
-
-        return outs
 
     def restart_resource_blocking(self, resource_name):
         '''
@@ -334,110 +327,112 @@ class Stack(object):
         except ValueError:
             raise exception.UserParameterMissing(key=key)
 
-    def resolve_static_refs(self, s):
+    def _resolve_static_refs(self, s):
         '''
             looking for { "Ref" : "str" }
         '''
-        if isinstance(s, dict):
+        def match(key, value):
-            for i in s:
+            return (key == 'Ref' and
-                if i == 'Ref' and \
+                    isinstance(value, basestring) and
-                      isinstance(s[i], (basestring, unicode)) and \
+                    value in self.parms)
-                      s[i] in self.parms:
-                    return self.parameter_get(s[i])
-                else:
-                    s[i] = self.resolve_static_refs(s[i])
-        elif isinstance(s, list):
-            for index, item in enumerate(s):
-                #print 'resolve_static_refs %d %s' % (index, item)
-                s[index] = self.resolve_static_refs(item)
-        return s
 
-    def resolve_find_in_map(self, s):
+        def handle(ref):
-        '''
+            return self.parameter_get(ref)
-            looking for { "Fn::FindInMap": ["str", "str"] }
-        '''
-        if isinstance(s, dict):
-            for i in s:
-                if i == 'Fn::FindInMap':
-                    obj = self.maps
-                    if isinstance(s[i], list):
-                        #print 'map list: %s' % s[i]
-                        for index, item in enumerate(s[i]):
-                            if isinstance(item, dict):
-                                item = self.resolve_find_in_map(item)
-                                #print 'map item dict: %s' % (item)
-                            else:
-                                pass
-                                #print 'map item str: %s' % (item)
-                            obj = obj[item]
-                    else:
-                        obj = obj[s[i]]
-                    return obj
-                else:
-                    s[i] = self.resolve_find_in_map(s[i])
-        elif isinstance(s, list):
-            for index, item in enumerate(s):
-                s[index] = self.resolve_find_in_map(item)
-        return s
 
-    def resolve_attributes(self, s):
+        return _resolve(match, handle, s)
+
+    def _resolve_find_in_map(self, s):
+        def handle(args):
+            try:
+                name, key, value = args
+                return self.maps[name][key][value]
+            except (ValueError, TypeError) as ex:
+                raise KeyError(str(ex))
+
+        return _resolve(lambda k, v: k == 'Fn::FindInMap', handle, s)
+
+    def _resolve_attributes(self, s):
         '''
             looking for something like:
             { "Fn::GetAtt" : [ "DBInstance", "Endpoint.Address" ] }
         '''
-        if isinstance(s, dict):
+        def match_ref(key, value):
-            for i in s:
+            return key == 'Ref' and value in self.resources
-                if i == 'Ref' and s[i] in self.resources:
-                    return self.resources[s[i]].FnGetRefId()
-                elif i == 'Fn::GetAtt':
-                    resource_name = s[i][0]
-                    key_name = s[i][1]
-                    res = self.resources.get(resource_name)
-                    rc = None
-                    if res:
-                        return res.FnGetAtt(key_name)
-                    else:
-                        raise exception.InvalidTemplateAttribute(
-                                        resource=resource_name, key=key_name)
-                    return rc
-                else:
-                    s[i] = self.resolve_attributes(s[i])
-        elif isinstance(s, list):
-            for index, item in enumerate(s):
-                s[index] = self.resolve_attributes(item)
-        return s
 
-    def resolve_joins(self, s):
+        def handle_ref(arg):
-        '''
+            return self.resources[arg].FnGetRefId()
-            looking for { "Fn::join": []}
+
-        '''
+        def handle_getatt(args):
-        if isinstance(s, dict):
+            resource, att = args
-            for i in s:
-                if i == 'Fn::Join':
-                    j = None
             try:
-                        j = s[i][0].join(s[i][1])
+                return self.resources[resource].FnGetAtt(att)
-                    except Exception:
+            except KeyError:
-                        logger.error('Could not join %s' % str(s[i]))
+                raise exception.InvalidTemplateAttribute(resource=resource,
-                    return j
+                                                         key=att)
-                else:
-                    s[i] = self.resolve_joins(s[i])
-        elif isinstance(s, list):
-            for index, item in enumerate(s):
-                s[index] = self.resolve_joins(item)
-        return s
 
-    def resolve_base64(self, s):
+        return _resolve(lambda k, v: k == 'Fn::GetAtt', handle_getatt,
+                        _resolve(match_ref, handle_ref, s))
+
+    @staticmethod
+    def _resolve_joins(s):
         '''
-            looking for { "Fn::join": [] }
+            looking for { "Fn::Join" : [] }
         '''
-        if isinstance(s, dict):
+        def handle(args):
-            for i in s:
+            delim, strings = args
-                if i == 'Fn::Base64':
+            return delim.join(strings)
-                    return s[i]
+
+        return _resolve(lambda k, v: k == 'Fn::Join', handle, s)
+
+    @staticmethod
+    def _resolve_base64(s):
+        '''
+            looking for { "Fn::Base64" : "" }
+        '''
+        return _resolve(lambda k, v: k == 'Fn::Base64', lambda d: d, s)
+
+    def resolve_static_data(self, snippet):
+        return transform(snippet, [self._resolve_static_refs,
+                                   self._resolve_find_in_map])
+
+    def resolve_runtime_data(self, snippet):
+        return transform(snippet, [self._resolve_attributes,
+                                   self._resolve_joins,
+                                   self._resolve_base64])
+
+
+def transform(data, transformations):
+    '''
+    Apply each of the transformation functions in the supplied list to the data
+    in turn.
+    '''
+    for t in transformations:
+        data = t(data)
+    return data
+
+
+def _resolve(match, handle, snippet):
+    '''
+    Resolve constructs in a snippet of a template. The supplied match function
+    should return True if a particular key-value pair should be substituted,
+    and the handle function should return the correct substitution when passed
+    the argument list as parameters.
+
+    Returns a copy of the original snippet with the substitutions performed.
+    '''
+    recurse = lambda k: _resolve(match, handle, snippet[k])
+
+    if isinstance(snippet, dict):
+        should_handle = lambda k: match(k, snippet[k])
+        matches = itertools.imap(recurse,
+                                 itertools.ifilter(should_handle, snippet))
+        try:
+            args = next(matches)
+        except StopIteration:
+            # No matches
+            return dict((k, recurse(k)) for k in snippet)
         else:
-                    s[i] = self.resolve_base64(s[i])
+            return handle(args)
-        elif isinstance(s, list):
+    elif isinstance(snippet, list):
-            for index, item in enumerate(s):
+        return [recurse(i) for i in range(len(snippet))]
-                s[index] = self.resolve_base64(item)
+    return snippet
-        return s

heat/engine/resources.py

@@ -53,13 +53,13 @@ class Resource(object):
         return ResourceClass(name, json, stack)
 
     def __init__(self, name, json_snippet, stack):
-        self.t = json_snippet
         self.depends_on = []
         self.references = []
         self.stack = stack
         self.name = name
+        self.t = stack.resolve_static_data(json_snippet)
         self.properties = checkeddict.Properties(name, self.properties_schema)
-        if not 'Properties' in self.t:
+        if 'Properties' not in self.t:
             # make a dummy entry to prevent having to check all over the
             # place for it.
             self.t['Properties'] = {}
@@ -75,9 +75,6 @@ class Resource(object):
             self.id = None
         self._nova = {}
 
-        stack.resolve_static_refs(self.t)
-        stack.resolve_find_in_map(self.t)
-
     def nova(self, service_type='compute'):
         if service_type in self._nova:
             return self._nova[service_type]
@@ -98,26 +95,22 @@ class Resource(object):
                                                  service_name=service_name)
         return self._nova[service_type]
 
+    def calculate_properties(self):
+        template = self.stack.resolve_runtime_data(self.t)
+
+        for p, v in template['Properties'].items():
+            self.properties[p] = v
+
     def create(self):
         logger.info('creating %s name:%s' % (self.t['Type'], self.name))
-
+        self.calculate_properties()
-        self.stack.resolve_attributes(self.t)
-        self.stack.resolve_joins(self.t)
-        self.stack.resolve_base64(self.t)
-        for p in self.t['Properties']:
-            self.properties[p] = self.t['Properties'][p]
         self.properties.validate()
 
     def validate(self):
         logger.info('validating %s name:%s' % (self.t['Type'], self.name))
 
-        self.stack.resolve_attributes(self.t)
-        self.stack.resolve_joins(self.t)
-        self.stack.resolve_base64(self.t)
-
         try:
-            for p in self.t['Properties']:
+            self.calculate_properties()
-                self.properties[p] = self.t['Properties'][p]
         except ValueError as ex:
                 return {'Error': '%s' % str(ex)}
         self.properties.validate()
@@ -160,7 +153,8 @@ class Resource(object):
             ev['name'] = new_state
             ev['resource_status_reason'] = reason
             ev['resource_type'] = self.t['Type']
-            ev['resource_properties'] = self.t['Properties']
+            self.calculate_properties()
+            ev['resource_properties'] = dict(self.properties)
             try:
                 db_api.event_create(None, ev)
             except Exception as ex:
@@ -168,24 +162,10 @@ class Resource(object):
             self.state = new_state
 
     def delete(self):
-        self.reload()
         logger.info('deleting %s name:%s inst:%s db_id:%s' %
                     (self.t['Type'], self.name,
                      self.instance_id, str(self.id)))
 
-    def reload(self):
-        '''
-        The point of this function is to get the Resource instance back
-        into the state that it was just after it was created. So we
-        need to retrieve things like ipaddresses and other variables
-        used by FnGetAtt and FnGetRefId. classes inheriting from Resource
-        might need to override this, but still call it.
-        This is currently used by stack.get_outputs()
-        '''
-        logger.info('reloading %s name:%s instance_id:%s' %
-                    (self.t['Type'], self.name, self.instance_id))
-        self.stack.resolve_attributes(self.t)
-
     def FnGetRefId(self):
         '''
         http://docs.amazonwebservices.com/AWSCloudFormation/latest/UserGuide/ \

heat/tests/test_resources.py

@@ -59,9 +59,7 @@ class instancesTest(unittest.TestCase):
                                       t['Resources']['WebServer'], stack)
 
         instance.itype_oflavor['256 MB Server'] = '256 MB Server'
-        instance.stack.resolve_attributes(instance.t)
+        instance.t = instance.stack.resolve_runtime_data(instance.t)
-        instance.stack.resolve_joins(instance.t)
-        instance.stack.resolve_base64(instance.t)
 
         # need to resolve the template functions
         server_userdata = instance._build_userdata(
@@ -109,9 +107,7 @@ class instancesTest(unittest.TestCase):
                                       t['Resources']['WebServer'], stack)
 
         instance.itype_oflavor['256 MB Server'] = '256 MB Server'
-        instance.stack.resolve_attributes(instance.t)
+        instance.t = instance.stack.resolve_runtime_data(instance.t)
-        instance.stack.resolve_joins(instance.t)
-        instance.stack.resolve_base64(instance.t)
 
         # need to resolve the template functions
         server_userdata = instance._build_userdata(

heat/tests/test_stacks.py

@@ -45,11 +45,9 @@ class stacksTest(unittest.TestCase):
         instances.Instance.nova().AndReturn(self.fc)
         instance = stack.resources['WebServer']
         instance.itype_oflavor['m1.large'] = 'm1.large'
-        instance.stack.resolve_attributes(instance.t)
+        instance.calculate_properties()
-        instance.stack.resolve_joins(instance.t)
-        instance.stack.resolve_base64(instance.t)
         server_userdata = instance._build_userdata(
-                                instance.t['Properties']['UserData'])
+                                instance.properties['UserData'])
         self.m.StubOutWithMock(self.fc.servers, 'create')
         self.fc.servers.create(image=744, flavor=3, key_name='test',
                 name='WebServer', security_groups=None,

heat/tests/test_validate.py

@@ -223,9 +223,6 @@ class validateTest(unittest.TestCase):
 
         self.m.ReplayAll()
         volumeattach = stack.resources['MountPoint']
-        stack.resolve_attributes(volumeattach.t)
-        stack.resolve_joins(volumeattach.t)
-        stack.resolve_base64(volumeattach.t)
         assert(volumeattach.validate() is None)
 
     def test_validate_volumeattach_invalid(self):
@@ -241,9 +238,6 @@ class validateTest(unittest.TestCase):
 
         self.m.ReplayAll()
         volumeattach = stack.resources['MountPoint']
-        stack.resolve_attributes(volumeattach.t)
-        stack.resolve_joins(volumeattach.t)
-        stack.resolve_base64(volumeattach.t)
         assert(volumeattach.validate())
 
     def test_validate_ref_valid(self):