Cache dep_attrs for all resources in definitions

When the dep_attrs function was written, it was used only in convergence
after checking a single resource. However, now we also use it to generate
data for the ResourceProxy objects, which is often done for all resources
simultaneously. That means doing O(n^2) dep_attrs() calls, which can really
slow things down when there is a large number of resources with complex
properties (or metadata).

This change adds an all_dep_attrs() call, which runs as fast as dep_attrs()
on everything except get_attr functions and their arguments, but only needs
to be called once instead of once per resource. (The get_attr function can
in future override the default implementation of all_dep_attrs() to be as
efficient as dep_attrs() as well.) The resulting data is cached in the
ResourceDefinition or OutputDefinition so that subsequent calls to their
get_attr() methods with different (or the same) resource names will use the
existing data.

Change-Id: If95f4c04b841519ce3d7492211f2696588c0ed48
Partially-Implements: blueprint stack-definition
Closes-Bug: #1684272

heat/engine/function.py

@@ -80,6 +80,32 @@ class Function(object):
         """
         return dep_attrs(self.args, resource_name)
 
+    def all_dep_attrs(self):
+        """Return resource, attribute name pairs of all attributes referenced.
+
+        Return an iterator over the resource name, attribute name tuples of
+        all attributes that this function references.
+
+        The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to
+        indicate that all attributes of the resource are required.
+
+        By default this calls the dep_attrs() method, but subclasses can
+        override to provide a more efficient implementation.
+        """
+        # If we are using the default dep_attrs method then it will only
+        # return data from the args anyway
+        if type(self).dep_attrs == Function.dep_attrs:
+            return all_dep_attrs(self.args)
+
+        def res_dep_attrs(resource_name):
+            return six.moves.zip(itertools.repeat(resource_name),
+                                 self.dep_attrs(resource_name))
+
+        resource_names = self.stack.enabled_rsrc_names()
+
+        return itertools.chain.from_iterable(six.moves.map(res_dep_attrs,
+                                                           resource_names))
+
     def __reduce__(self):
         """Return a representation of the function suitable for pickling.
 
@@ -189,6 +215,25 @@ class Macro(Function):
         """
         return dep_attrs(self.parsed, resource_name)
 
+    def all_dep_attrs(self):
+        """Return resource, attribute name pairs of all attributes referenced.
+
+        Return an iterator over the resource name, attribute name tuples of
+        all attributes that this function references.
+
+        The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to
+        indicate that all attributes of the resource are required.
+
+        By default this calls the dep_attrs() method, but subclasses can
+        override to provide a more efficient implementation.
+        """
+        # If we are using the default dep_attrs method then it will only
+        # return data from the transformed parsed args anyway
+        if type(self).dep_attrs == Macro.dep_attrs:
+            return all_dep_attrs(self.parsed)
+
+        return super(Macro, self).all_dep_attrs()
+
     def __reduce__(self):
         """Return a representation of the macro result suitable for pickling.
 
@@ -299,6 +344,29 @@ def dep_attrs(snippet, resource_name):
     return []
 
 
+def all_dep_attrs(snippet):
+    """Iterator over resource, attribute name pairs referenced in a snippet.
+
+    The snippet should be already parsed to insert Function objects where
+    appropriate.
+
+    :returns: an iterator over the resource name, attribute name tuples of all
+    attributes that are referenced in the template snippet.
+    """
+
+    if isinstance(snippet, Function):
+        return snippet.all_dep_attrs()
+
+    elif isinstance(snippet, collections.Mapping):
+        res_attrs = (all_dep_attrs(value) for value in snippet.values())
+        return itertools.chain.from_iterable(res_attrs)
+    elif (not isinstance(snippet, six.string_types) and
+          isinstance(snippet, collections.Iterable)):
+        res_attrs = (all_dep_attrs(value) for value in snippet)
+        return itertools.chain.from_iterable(res_attrs)
+    return []
+
+
 class Invalid(Function):
     """A function for checking condition functions and to force failures.
 

heat/engine/output.py

@@ -11,6 +11,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import collections
 import copy
 import six
 
@@ -27,6 +28,7 @@ class OutputDefinition(object):
         self._resolved_value = None
         self._description = description
         self._deps = None
+        self._all_dep_attrs = None
 
     def validate(self, path=''):
         """Validate the output value without resolving it."""
@@ -44,12 +46,21 @@ class OutputDefinition(object):
                 self._deps = set()
         return self._deps
 
-    def dep_attrs(self, resource_name):
+    def dep_attrs(self, resource_name, load_all=False):
         """Iterate over attributes of a given resource that this references.
 
         Return an iterator over dependent attributes for specified
         resource_name in the output's value field.
         """
+        if self._all_dep_attrs is None and load_all:
+            attr_map = collections.defaultdict(set)
+            for r, a in function.all_dep_attrs(self._value):
+                attr_map[r].add(a)
+            self._all_dep_attrs = attr_map
+
+        if self._all_dep_attrs is not None:
+            return iter(self._all_dep_attrs.get(resource_name, []))
+
         return function.dep_attrs(self._value, resource_name)
 
     def get_value(self):

heat/engine/resource.py

@@ -948,7 +948,8 @@ class Resource(status.ResourceStatus):
             self.attributes.reset_resolved_values()
 
     def referenced_attrs(self, stk_defn=None,
-                         in_resources=True, in_outputs=True):
+                         in_resources=True, in_outputs=True,
+                         load_all=False):
         """Return the set of all attributes referenced in the template.
 
         This enables the resource to calculate which of its attributes will
@@ -966,7 +967,8 @@ class Resource(status.ResourceStatus):
             stk_defn = self.stack.defn
 
         def get_dep_attrs(source):
-            return set(itertools.chain.from_iterable(s.dep_attrs(self.name)
+            return set(itertools.chain.from_iterable(s.dep_attrs(self.name,
+                                                                 load_all)
                                                      for s in source))
 
         refd_attrs = set()
@@ -1030,13 +1032,16 @@ class Resource(status.ResourceStatus):
                     except exception.InvalidTemplateAttribute as ita:
                         LOG.info('%s', ita)
 
+        load_all = not self.stack.in_convergence_check
         dep_attrs = self.referenced_attrs(stk_defn,
                                           in_resources=for_resources,
-                                          in_outputs=for_outputs)
+                                          in_outputs=for_outputs,
+                                          load_all=load_all)
 
         # Ensure all attributes referenced in outputs get cached
         if for_outputs is False and self.stack.convergence:
-            out_attrs = self.referenced_attrs(stk_defn, in_resources=False)
+            out_attrs = self.referenced_attrs(stk_defn, in_resources=False,
+                                              load_all=load_all)
             for e in get_attrs(out_attrs - dep_attrs, cacheable_only=True):
                 pass
 

heat/engine/rsrc_defn.py

@@ -100,6 +100,7 @@ class ResourceDefinition(object):
         self._hash = hash(self.resource_type)
         self._rendering = None
         self._dep_names = None
+        self._all_dep_attrs = None
 
         assert isinstance(self.description, six.string_types)
 
@@ -192,12 +193,23 @@ class ResourceDefinition(object):
             external_id=reparse_snippet(self._external_id),
             condition=self._condition)
 
-    def dep_attrs(self, resource_name):
+    def dep_attrs(self, resource_name, load_all=False):
         """Iterate over attributes of a given resource that this references.
 
         Return an iterator over dependent attributes for specified
         resource_name in resources' properties and metadata fields.
         """
+        if self._all_dep_attrs is None and load_all:
+            attr_map = collections.defaultdict(set)
+            atts = itertools.chain(function.all_dep_attrs(self._properties),
+                                   function.all_dep_attrs(self._metadata))
+            for res_name, att_name in atts:
+                attr_map[res_name].add(att_name)
+            self._all_dep_attrs = attr_map
+
+        if self._all_dep_attrs is not None:
+            return self._all_dep_attrs[resource_name]
+
         return itertools.chain(function.dep_attrs(self._properties,
                                                   resource_name),
                                function.dep_attrs(self._metadata,

heat/engine/stk_defn.py

@@ -11,6 +11,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import itertools
 import six
 
 from heat.common import exception
@@ -245,6 +246,19 @@ def update_resource_data(stack_definition, resource_name, resource_data):
     stack_definition._resource_data[resource_name] = resource_data
     stack_definition._resources.pop(resource_name, None)
 
+    # Clear the cached dep_attrs for any resource or output that directly
+    # depends on the resource whose data we are updating. This ensures that if
+    # any of the data we just updated is referenced in the path of a get_attr
+    # function, future calls to dep_attrs() will reflect this new data.
+    res_defns = stack_definition._resource_defns or {}
+    op_defns = stack_definition._output_defns or {}
+
+    all_defns = itertools.chain(six.itervalues(res_defns),
+                                six.itervalues(op_defns))
+    for defn in all_defns:
+        if resource_name in defn.required_resource_names():
+            defn._all_dep_attrs = None
+
 
 def add_resource(stack_definition, resource_definition):
     """Insert the given resource definition into the stack definition.

heat/tests/test_stack_collect_attributes.py

@@ -224,18 +224,26 @@ class DepAttrsTest(common.HeatTestCase):
         super(DepAttrsTest, self).setUp()
         self.ctx = utils.dummy_context()
 
-    def test_dep_attrs(self):
-        parsed_tmpl = template_format.parse(self.tmpl)
+        self.parsed_tmpl = template_format.parse(self.tmpl)
         self.stack = stack.Stack(self.ctx, 'test_stack',
-                                 template.Template(parsed_tmpl))
+                                 template.Template(self.parsed_tmpl))
 
+    def test_dep_attrs(self):
         for res in six.itervalues(self.stack):
             definitions = (self.stack.defn.resource_definition(n)
-                           for n in parsed_tmpl['resources'])
+                           for n in self.parsed_tmpl['resources'])
             self.assertEqual(self.expected[res.name],
                              set(itertools.chain.from_iterable(
                                  d.dep_attrs(res.name) for d in definitions)))
 
+    def test_all_dep_attrs(self):
+        for res in six.itervalues(self.stack):
+            definitions = (self.stack.defn.resource_definition(n)
+                           for n in self.parsed_tmpl['resources'])
+            attrs = set(itertools.chain.from_iterable(
+                d.dep_attrs(res.name, load_all=True) for d in definitions))
+            self.assertEqual(self.expected[res.name], attrs)
+
 
 class ReferencedAttrsTest(common.HeatTestCase):
     def setUp(self):