Convergence: Allow creating lightweight stacks

Currently, we load all the resources from the database to resolve
template functions of dependent resources. In convergence, because
every worker will load it's own stack object, loading all the
resources for every resource lifecycle operation will be inefficient.

This patch allows creating lightweight stacks using a cache provided
which will never query the database and only depend on the template
and the cache provided. Function resolutions will now happen by querying
the values in the cache provided if it exists else None is returned.
The resultant lightweight stack will effectively be the stack attributes
loaded from the database and it's raw template with the dependent
resource's functions resolved which will serve as the input for
individual resource objects that will be worked upon.

blueprint convergence-lightweight-stack

Change-Id: I6dbaa7ee4e9d534c31823b4812efcb387c695a22

heat/engine/cfn/functions.py

@@ -198,6 +198,10 @@ class GetAtt(function.Function):
         r = self._resource()
         if (r.action in (r.CREATE, r.ADOPT, r.SUSPEND, r.RESUME, r.UPDATE)):
             return r.FnGetAtt(attribute)
+        # NOTE(sirushtim): Add r.INIT to states above once convergence
+        # is the default.
+        elif r.stack.has_cache_data() and r.action == r.INIT:
+            return r.FnGetAtt(attribute)
         else:
             return None
 

heat/engine/resource.py

@@ -180,9 +180,10 @@ class Resource(object):
         self.replaced_by = None
         self.current_template_id = None
 
-        resource = stack.db_resource_get(name)
-        if resource:
-            self._load_data(resource)
+        if not stack.has_cache_data():
+            resource = stack.db_resource_get(name)
+            if resource:
+                self._load_data(resource)
 
     def rpc_client(self):
         '''Return a client for making engine RPC calls.'''
@@ -1091,6 +1092,9 @@ class Resource(object):
 
         :results: the id or name of the resource.
         '''
+        if self.stack.has_cache_data():
+            return self.stack.cache_data_resource_id(self.name)
+
         if self.resource_id is not None:
             return six.text_type(self.resource_id)
         else:
@@ -1111,13 +1115,18 @@ class Resource(object):
         :param path: a list of path components to select from the attribute.
         :returns: the attribute value.
         '''
-        try:
-            attribute = self.attributes[key]
-        except KeyError:
-            raise exception.InvalidTemplateAttribute(resource=self.name,
-                                                     key=key)
+        if self.stack.has_cache_data():
+            # Load from cache for lightweight resources.
+            attribute = self.stack.cache_data_resource_attribute(
+                self.name, key)
         else:
-            return attributes.select_from_attribute(attribute, path)
+            try:
+                attribute = self.attributes[key]
+            except KeyError:
+                raise exception.InvalidTemplateAttribute(resource=self.name,
+                                                         key=key)
+
+        return attributes.select_from_attribute(attribute, path)
 
     def FnBase64(self, data):
         '''

heat/engine/stack.py

@@ -89,11 +89,16 @@ class Stack(collections.Mapping):
                  use_stored_context=False, username=None,
                  nested_depth=0, strict_validate=True, convergence=False,
                  current_traversal=None, tags=None, prev_raw_template_id=None,
-                 current_deps=None):
+                 current_deps=None, cache_data=None):
+
         '''
         Initialise from a context, name, Template object and (optionally)
         Environment object. The database ID may also be initialised, if the
         stack is already in the database.
+
+        Creating a stack with cache_data creates a lightweight stack which
+        will not load any resources from the database and resolve the
+        functions from the cache_data specified.
         '''
 
         def _validate_stack_name(name):
@@ -135,6 +140,7 @@ class Stack(collections.Mapping):
         self.tags = tags
         self.prev_raw_template_id = prev_raw_template_id
         self.current_deps = current_deps
+        self.cache_data = cache_data
 
         if use_stored_context:
             self.context = self.stored_context()
@@ -339,8 +345,8 @@ class Stack(collections.Mapping):
         return deps
 
     @classmethod
-    def load(cls, context, stack_id=None, stack=None,
-             show_deleted=True, use_stored_context=False, force_reload=False):
+    def load(cls, context, stack_id=None, stack=None, show_deleted=True,
+             use_stored_context=False, force_reload=False, cache_data=None):
         '''Retrieve a Stack from the database.'''
         if stack is None:
             stack = stack_object.Stack.get_by_id(
@@ -356,7 +362,8 @@ class Stack(collections.Mapping):
             stack.refresh()
 
         return cls._from_db(context, stack,
-                            use_stored_context=use_stored_context)
+                            use_stored_context=use_stored_context,
+                            cache_data=cache_data)
 
     @classmethod
     def load_all(cls, context, limit=None, marker=None, sort_keys=None,
@@ -384,7 +391,7 @@ class Stack(collections.Mapping):
 
     @classmethod
     def _from_db(cls, context, stack, resolve_data=True,
-                 use_stored_context=False):
+                 use_stored_context=False, cache_data=None):
         template = tmpl.Template.load(
             context, stack.raw_template_id, stack.raw_template)
         tags = None
@@ -407,7 +414,7 @@ class Stack(collections.Mapping):
                    username=stack.username, convergence=stack.convergence,
                    current_traversal=stack.current_traversal, tags=tags,
                    prev_raw_template_id=stack.prev_raw_template_id,
-                   current_deps=stack.current_deps)
+                   current_deps=stack.current_deps, cache_data=cache_data)
 
     def get_kwargs_for_cloning(self, keep_status=False, only_db=False):
         """Get common kwargs for calling Stack() for cloning.
@@ -1572,3 +1579,16 @@ class Stack(collections.Mapping):
         # of other resources, so ensure that attributes are re-calculated
         for res in six.itervalues(self.resources):
             res.attributes.reset_resolved_values()
+
+    def has_cache_data(self):
+        if self.cache_data is not None:
+            return True
+
+        return False
+
+    def cache_data_resource_id(self, resource_name):
+        return self.cache_data.get(resource_name, {}).get('id')
+
+    def cache_data_resource_attribute(self, resource_name, attribute_key):
+        return self.cache_data.get(
+            resource_name, {}).get('attributes', {}).get(attribute_key)

heat/tests/neutron/test_neutron.py

@@ -109,6 +109,7 @@ class NeutronTest(common.HeatTestCase):
 
         tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
         stack = mock.MagicMock()
+        stack.has_cache_data = mock.Mock(return_value=False)
         res = SomeNeutronResource('aresource', tmpl, stack)
 
         mock_show_resource = mock.MagicMock()

heat/tests/test_stack.py

@@ -232,6 +232,37 @@ class StackTest(common.HeatTestCase):
         all_resources = list(self.stack.iter_resources(1))
         self.assertEqual(5, len(all_resources))
 
+    @mock.patch.object(stack.Stack, 'db_resource_get')
+    def test_iter_resources_cached(self, mock_drg):
+        tpl = {'HeatTemplateFormatVersion': '2012-12-12',
+               'Resources':
+               {'A': {'Type': 'GenericResourceType'},
+                'B': {'Type': 'GenericResourceType'}}}
+        self.stack = stack.Stack(self.ctx, 'test_stack',
+                                 template.Template(tpl),
+                                 status_reason='blarg',
+                                 cache_data={})
+
+        def get_more(nested_depth=0):
+            yield 'X'
+            yield 'Y'
+            yield 'Z'
+
+        self.stack['A'].nested = mock.MagicMock()
+        self.stack['A'].nested.return_value.iter_resources = mock.MagicMock(
+            side_effect=get_more)
+
+        resource_generator = self.stack.iter_resources()
+        self.assertIsNot(resource_generator, list)
+
+        first_level_resources = list(resource_generator)
+        self.assertEqual(2, len(first_level_resources))
+        all_resources = list(self.stack.iter_resources(1))
+        self.assertEqual(5, len(all_resources))
+
+        # A cache supplied means we should never query the database.
+        self.assertFalse(mock_drg.called)
+
     def test_root_stack_no_parent(self):
         tpl = {'HeatTemplateFormatVersion': '2012-12-12',
                'Resources':
@@ -295,7 +326,7 @@ class StackTest(common.HeatTestCase):
                              current_traversal=None,
                              tags=mox.IgnoreArg(),
                              prev_raw_template_id=None,
-                             current_deps=None)
+                             current_deps=None, cache_data=None)
 
         self.m.ReplayAll()
         stack.Stack.load(self.ctx, stack_id=self.stack.id)
@@ -1853,6 +1884,72 @@ class StackTest(common.HeatTestCase):
         self.assertEqual(
             'foo', self.stack.resources['A'].properties['a_string'])
 
+    @mock.patch.object(stack.Stack, 'db_resource_get')
+    def test_lightweight_stack_getatt(self, mock_drg):
+        tmpl = template.Template({
+            'HeatTemplateFormatVersion': '2012-12-12',
+            'Resources': {
+                'foo': {'Type': 'GenericResourceType'},
+                'bar': {
+                    'Type': 'ResourceWithPropsType',
+                    'Properties': {
+                        'Foo': {'Fn::GetAtt': ['foo', 'bar']},
+                    }
+                }
+            }
+        })
+
+        cache_data = {'foo': {'attributes': {'bar': 'baz'}}}
+        tmpl_stack = stack.Stack(self.ctx, 'test', tmpl)
+        tmpl_stack.store()
+        lightweight_stack = stack.Stack.load(self.ctx, stack_id=tmpl_stack.id,
+                                             cache_data=cache_data)
+
+        # Check if the property has the appropriate resolved value.
+        cached_property = lightweight_stack['bar'].properties['Foo']
+        self.assertEqual(cached_property, 'baz')
+
+        # Make sure FnGetAtt returns the cached value.
+        attr_value = lightweight_stack['foo'].FnGetAtt('bar')
+        self.assertEqual('baz', attr_value)
+
+        # Make sure calls are not made to the database to retrieve the
+        # resource state.
+        self.assertFalse(mock_drg.called)
+
+    @mock.patch.object(stack.Stack, 'db_resource_get')
+    def test_lightweight_stack_getrefid(self, mock_drg):
+        tmpl = template.Template({
+            'HeatTemplateFormatVersion': '2012-12-12',
+            'Resources': {
+                'foo': {'Type': 'GenericResourceType'},
+                'bar': {
+                    'Type': 'ResourceWithPropsType',
+                    'Properties': {
+                        'Foo': {'Ref': 'foo'},
+                    }
+                }
+            }
+        })
+
+        cache_data = {'foo': {'id': 'physical-resource-id'}}
+        tmpl_stack = stack.Stack(self.ctx, 'test', tmpl)
+        tmpl_stack.store()
+        lightweight_stack = stack.Stack.load(self.ctx, stack_id=tmpl_stack.id,
+                                             cache_data=cache_data)
+
+        # Check if the property has the appropriate resolved value.
+        cached_property = lightweight_stack['bar'].properties['Foo']
+        self.assertEqual(cached_property, 'physical-resource-id')
+
+        # Make sure FnGetRefId returns the cached value.
+        resource_id = lightweight_stack['foo'].FnGetRefId()
+        self.assertEqual('physical-resource-id', resource_id)
+
+        # Make sure calls are not made to the database to retrieve the
+        # resource state.
+        self.assertFalse(mock_drg.called)
+
 
 class StackKwargsForCloningTest(common.HeatTestCase):
     scenarios = [