Cache dep_attrs for all resources in definitions

When the dep_attrs function was written, it was used only in convergence
after checking a single resource. However, now we also use it to generate
data for the ResourceProxy objects, which is often done for all resources
simultaneously. That means doing O(n^2) dep_attrs() calls, which can really
slow things down when there is a large number of resources with complex
properties (or metadata).

This change adds an all_dep_attrs() call, which runs as fast as dep_attrs()
on everything except get_attr functions and their arguments, but only needs
to be called once instead of once per resource. (The get_attr function can
in future override the default implementation of all_dep_attrs() to be as
efficient as dep_attrs() as well.) The resulting data is cached in the
ResourceDefinition or OutputDefinition so that subsequent calls to their
get_attr() methods with different (or the same) resource names will use the
existing data.

Change-Id: If95f4c04b841519ce3d7492211f2696588c0ed48
Partially-Implements: blueprint stack-definition
Closes-Bug: #1684272
changes/46/477046/19
Zane Bitter 5 years ago
parent b0916ad5bb
commit 3c13cb82a6
  1. 68
      heat/engine/function.py
  2. 13
      heat/engine/output.py
  3. 13
      heat/engine/resource.py
  4. 14
      heat/engine/rsrc_defn.py
  5. 14
      heat/engine/stk_defn.py
  6. 16
      heat/tests/test_stack_collect_attributes.py

@ -80,6 +80,32 @@ class Function(object):
"""
return dep_attrs(self.args, resource_name)
def all_dep_attrs(self):
"""Return resource, attribute name pairs of all attributes referenced.
Return an iterator over the resource name, attribute name tuples of
all attributes that this function references.
The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to
indicate that all attributes of the resource are required.
By default this calls the dep_attrs() method, but subclasses can
override to provide a more efficient implementation.
"""
# If we are using the default dep_attrs method then it will only
# return data from the args anyway
if type(self).dep_attrs == Function.dep_attrs:
return all_dep_attrs(self.args)
def res_dep_attrs(resource_name):
return six.moves.zip(itertools.repeat(resource_name),
self.dep_attrs(resource_name))
resource_names = self.stack.enabled_rsrc_names()
return itertools.chain.from_iterable(six.moves.map(res_dep_attrs,
resource_names))
def __reduce__(self):
"""Return a representation of the function suitable for pickling.
@ -189,6 +215,25 @@ class Macro(Function):
"""
return dep_attrs(self.parsed, resource_name)
def all_dep_attrs(self):
"""Return resource, attribute name pairs of all attributes referenced.
Return an iterator over the resource name, attribute name tuples of
all attributes that this function references.
The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to
indicate that all attributes of the resource are required.
By default this calls the dep_attrs() method, but subclasses can
override to provide a more efficient implementation.
"""
# If we are using the default dep_attrs method then it will only
# return data from the transformed parsed args anyway
if type(self).dep_attrs == Macro.dep_attrs:
return all_dep_attrs(self.parsed)
return super(Macro, self).all_dep_attrs()
def __reduce__(self):
"""Return a representation of the macro result suitable for pickling.
@ -299,6 +344,29 @@ def dep_attrs(snippet, resource_name):
return []
def all_dep_attrs(snippet):
"""Iterator over resource, attribute name pairs referenced in a snippet.
The snippet should be already parsed to insert Function objects where
appropriate.
:returns: an iterator over the resource name, attribute name tuples of all
attributes that are referenced in the template snippet.
"""
if isinstance(snippet, Function):
return snippet.all_dep_attrs()
elif isinstance(snippet, collections.Mapping):
res_attrs = (all_dep_attrs(value) for value in snippet.values())
return itertools.chain.from_iterable(res_attrs)
elif (not isinstance(snippet, six.string_types) and
isinstance(snippet, collections.Iterable)):
res_attrs = (all_dep_attrs(value) for value in snippet)
return itertools.chain.from_iterable(res_attrs)
return []
class Invalid(Function):
"""A function for checking condition functions and to force failures.

@ -11,6 +11,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import copy
import six
@ -27,6 +28,7 @@ class OutputDefinition(object):
self._resolved_value = None
self._description = description
self._deps = None
self._all_dep_attrs = None
def validate(self, path=''):
"""Validate the output value without resolving it."""
@ -44,12 +46,21 @@ class OutputDefinition(object):
self._deps = set()
return self._deps
def dep_attrs(self, resource_name):
def dep_attrs(self, resource_name, load_all=False):
"""Iterate over attributes of a given resource that this references.
Return an iterator over dependent attributes for specified
resource_name in the output's value field.
"""
if self._all_dep_attrs is None and load_all:
attr_map = collections.defaultdict(set)
for r, a in function.all_dep_attrs(self._value):
attr_map[r].add(a)
self._all_dep_attrs = attr_map
if self._all_dep_attrs is not None:
return iter(self._all_dep_attrs.get(resource_name, []))
return function.dep_attrs(self._value, resource_name)
def get_value(self):

@ -948,7 +948,8 @@ class Resource(status.ResourceStatus):
self.attributes.reset_resolved_values()
def referenced_attrs(self, stk_defn=None,
in_resources=True, in_outputs=True):
in_resources=True, in_outputs=True,
load_all=False):
"""Return the set of all attributes referenced in the template.
This enables the resource to calculate which of its attributes will
@ -966,7 +967,8 @@ class Resource(status.ResourceStatus):
stk_defn = self.stack.defn
def get_dep_attrs(source):
return set(itertools.chain.from_iterable(s.dep_attrs(self.name)
return set(itertools.chain.from_iterable(s.dep_attrs(self.name,
load_all)
for s in source))
refd_attrs = set()
@ -1030,13 +1032,16 @@ class Resource(status.ResourceStatus):
except exception.InvalidTemplateAttribute as ita:
LOG.info('%s', ita)
load_all = not self.stack.in_convergence_check
dep_attrs = self.referenced_attrs(stk_defn,
in_resources=for_resources,
in_outputs=for_outputs)
in_outputs=for_outputs,
load_all=load_all)
# Ensure all attributes referenced in outputs get cached
if for_outputs is False and self.stack.convergence:
out_attrs = self.referenced_attrs(stk_defn, in_resources=False)
out_attrs = self.referenced_attrs(stk_defn, in_resources=False,
load_all=load_all)
for e in get_attrs(out_attrs - dep_attrs, cacheable_only=True):
pass

@ -100,6 +100,7 @@ class ResourceDefinition(object):
self._hash = hash(self.resource_type)
self._rendering = None
self._dep_names = None
self._all_dep_attrs = None
assert isinstance(self.description, six.string_types)
@ -192,12 +193,23 @@ class ResourceDefinition(object):
external_id=reparse_snippet(self._external_id),
condition=self._condition)
def dep_attrs(self, resource_name):
def dep_attrs(self, resource_name, load_all=False):
"""Iterate over attributes of a given resource that this references.
Return an iterator over dependent attributes for specified
resource_name in resources' properties and metadata fields.
"""
if self._all_dep_attrs is None and load_all:
attr_map = collections.defaultdict(set)
atts = itertools.chain(function.all_dep_attrs(self._properties),
function.all_dep_attrs(self._metadata))
for res_name, att_name in atts:
attr_map[res_name].add(att_name)
self._all_dep_attrs = attr_map
if self._all_dep_attrs is not None:
return self._all_dep_attrs[resource_name]
return itertools.chain(function.dep_attrs(self._properties,
resource_name),
function.dep_attrs(self._metadata,

@ -11,6 +11,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import six
from heat.common import exception
@ -245,6 +246,19 @@ def update_resource_data(stack_definition, resource_name, resource_data):
stack_definition._resource_data[resource_name] = resource_data
stack_definition._resources.pop(resource_name, None)
# Clear the cached dep_attrs for any resource or output that directly
# depends on the resource whose data we are updating. This ensures that if
# any of the data we just updated is referenced in the path of a get_attr
# function, future calls to dep_attrs() will reflect this new data.
res_defns = stack_definition._resource_defns or {}
op_defns = stack_definition._output_defns or {}
all_defns = itertools.chain(six.itervalues(res_defns),
six.itervalues(op_defns))
for defn in all_defns:
if resource_name in defn.required_resource_names():
defn._all_dep_attrs = None
def add_resource(stack_definition, resource_definition):
"""Insert the given resource definition into the stack definition.

@ -224,18 +224,26 @@ class DepAttrsTest(common.HeatTestCase):
super(DepAttrsTest, self).setUp()
self.ctx = utils.dummy_context()
def test_dep_attrs(self):
parsed_tmpl = template_format.parse(self.tmpl)
self.parsed_tmpl = template_format.parse(self.tmpl)
self.stack = stack.Stack(self.ctx, 'test_stack',
template.Template(parsed_tmpl))
template.Template(self.parsed_tmpl))
def test_dep_attrs(self):
for res in six.itervalues(self.stack):
definitions = (self.stack.defn.resource_definition(n)
for n in parsed_tmpl['resources'])
for n in self.parsed_tmpl['resources'])
self.assertEqual(self.expected[res.name],
set(itertools.chain.from_iterable(
d.dep_attrs(res.name) for d in definitions)))
def test_all_dep_attrs(self):
for res in six.itervalues(self.stack):
definitions = (self.stack.defn.resource_definition(n)
for n in self.parsed_tmpl['resources'])
attrs = set(itertools.chain.from_iterable(
d.dep_attrs(res.name, load_all=True) for d in definitions))
self.assertEqual(self.expected[res.name], attrs)
class ReferencedAttrsTest(common.HeatTestCase):
def setUp(self):

Loading…
Cancel
Save