diff --git a/heat/engine/function.py b/heat/engine/function.py index 4fb6047bcf..5b1a63e636 100644 --- a/heat/engine/function.py +++ b/heat/engine/function.py @@ -80,6 +80,32 @@ class Function(object): """ return dep_attrs(self.args, resource_name) + def all_dep_attrs(self): + """Return resource, attribute name pairs of all attributes referenced. + + Return an iterator over the resource name, attribute name tuples of + all attributes that this function references. + + The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to + indicate that all attributes of the resource are required. + + By default this calls the dep_attrs() method, but subclasses can + override to provide a more efficient implementation. + """ + # If we are using the default dep_attrs method then it will only + # return data from the args anyway + if type(self).dep_attrs == Function.dep_attrs: + return all_dep_attrs(self.args) + + def res_dep_attrs(resource_name): + return six.moves.zip(itertools.repeat(resource_name), + self.dep_attrs(resource_name)) + + resource_names = self.stack.enabled_rsrc_names() + + return itertools.chain.from_iterable(six.moves.map(res_dep_attrs, + resource_names)) + def __reduce__(self): """Return a representation of the function suitable for pickling. @@ -189,6 +215,25 @@ class Macro(Function): """ return dep_attrs(self.parsed, resource_name) + def all_dep_attrs(self): + """Return resource, attribute name pairs of all attributes referenced. + + Return an iterator over the resource name, attribute name tuples of + all attributes that this function references. + + The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to + indicate that all attributes of the resource are required. + + By default this calls the dep_attrs() method, but subclasses can + override to provide a more efficient implementation. + """ + # If we are using the default dep_attrs method then it will only + # return data from the transformed parsed args anyway + if type(self).dep_attrs == Macro.dep_attrs: + return all_dep_attrs(self.parsed) + + return super(Macro, self).all_dep_attrs() + def __reduce__(self): """Return a representation of the macro result suitable for pickling. @@ -299,6 +344,29 @@ def dep_attrs(snippet, resource_name): return [] +def all_dep_attrs(snippet): + """Iterator over resource, attribute name pairs referenced in a snippet. + + The snippet should be already parsed to insert Function objects where + appropriate. + + :returns: an iterator over the resource name, attribute name tuples of all + attributes that are referenced in the template snippet. + """ + + if isinstance(snippet, Function): + return snippet.all_dep_attrs() + + elif isinstance(snippet, collections.Mapping): + res_attrs = (all_dep_attrs(value) for value in snippet.values()) + return itertools.chain.from_iterable(res_attrs) + elif (not isinstance(snippet, six.string_types) and + isinstance(snippet, collections.Iterable)): + res_attrs = (all_dep_attrs(value) for value in snippet) + return itertools.chain.from_iterable(res_attrs) + return [] + + class Invalid(Function): """A function for checking condition functions and to force failures. diff --git a/heat/engine/output.py b/heat/engine/output.py index b8e82010a0..73c41e62fa 100644 --- a/heat/engine/output.py +++ b/heat/engine/output.py @@ -11,6 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import copy import six @@ -27,6 +28,7 @@ class OutputDefinition(object): self._resolved_value = None self._description = description self._deps = None + self._all_dep_attrs = None def validate(self, path=''): """Validate the output value without resolving it.""" @@ -44,12 +46,21 @@ class OutputDefinition(object): self._deps = set() return self._deps - def dep_attrs(self, resource_name): + def dep_attrs(self, resource_name, load_all=False): """Iterate over attributes of a given resource that this references. Return an iterator over dependent attributes for specified resource_name in the output's value field. """ + if self._all_dep_attrs is None and load_all: + attr_map = collections.defaultdict(set) + for r, a in function.all_dep_attrs(self._value): + attr_map[r].add(a) + self._all_dep_attrs = attr_map + + if self._all_dep_attrs is not None: + return iter(self._all_dep_attrs.get(resource_name, [])) + return function.dep_attrs(self._value, resource_name) def get_value(self): diff --git a/heat/engine/resource.py b/heat/engine/resource.py index e0c70096de..a53c9f28f1 100644 --- a/heat/engine/resource.py +++ b/heat/engine/resource.py @@ -948,7 +948,8 @@ class Resource(status.ResourceStatus): self.attributes.reset_resolved_values() def referenced_attrs(self, stk_defn=None, - in_resources=True, in_outputs=True): + in_resources=True, in_outputs=True, + load_all=False): """Return the set of all attributes referenced in the template. This enables the resource to calculate which of its attributes will @@ -966,7 +967,8 @@ class Resource(status.ResourceStatus): stk_defn = self.stack.defn def get_dep_attrs(source): - return set(itertools.chain.from_iterable(s.dep_attrs(self.name) + return set(itertools.chain.from_iterable(s.dep_attrs(self.name, + load_all) for s in source)) refd_attrs = set() @@ -1030,13 +1032,16 @@ class Resource(status.ResourceStatus): except exception.InvalidTemplateAttribute as ita: LOG.info('%s', ita) + load_all = not self.stack.in_convergence_check dep_attrs = self.referenced_attrs(stk_defn, in_resources=for_resources, - in_outputs=for_outputs) + in_outputs=for_outputs, + load_all=load_all) # Ensure all attributes referenced in outputs get cached if for_outputs is False and self.stack.convergence: - out_attrs = self.referenced_attrs(stk_defn, in_resources=False) + out_attrs = self.referenced_attrs(stk_defn, in_resources=False, + load_all=load_all) for e in get_attrs(out_attrs - dep_attrs, cacheable_only=True): pass diff --git a/heat/engine/rsrc_defn.py b/heat/engine/rsrc_defn.py index 295ac448c3..5ae3d7f25c 100644 --- a/heat/engine/rsrc_defn.py +++ b/heat/engine/rsrc_defn.py @@ -100,6 +100,7 @@ class ResourceDefinition(object): self._hash = hash(self.resource_type) self._rendering = None self._dep_names = None + self._all_dep_attrs = None assert isinstance(self.description, six.string_types) @@ -192,12 +193,23 @@ class ResourceDefinition(object): external_id=reparse_snippet(self._external_id), condition=self._condition) - def dep_attrs(self, resource_name): + def dep_attrs(self, resource_name, load_all=False): """Iterate over attributes of a given resource that this references. Return an iterator over dependent attributes for specified resource_name in resources' properties and metadata fields. """ + if self._all_dep_attrs is None and load_all: + attr_map = collections.defaultdict(set) + atts = itertools.chain(function.all_dep_attrs(self._properties), + function.all_dep_attrs(self._metadata)) + for res_name, att_name in atts: + attr_map[res_name].add(att_name) + self._all_dep_attrs = attr_map + + if self._all_dep_attrs is not None: + return self._all_dep_attrs[resource_name] + return itertools.chain(function.dep_attrs(self._properties, resource_name), function.dep_attrs(self._metadata, diff --git a/heat/engine/stk_defn.py b/heat/engine/stk_defn.py index 17aadd9707..286ab5b1aa 100644 --- a/heat/engine/stk_defn.py +++ b/heat/engine/stk_defn.py @@ -11,6 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. +import itertools import six from heat.common import exception @@ -245,6 +246,19 @@ def update_resource_data(stack_definition, resource_name, resource_data): stack_definition._resource_data[resource_name] = resource_data stack_definition._resources.pop(resource_name, None) + # Clear the cached dep_attrs for any resource or output that directly + # depends on the resource whose data we are updating. This ensures that if + # any of the data we just updated is referenced in the path of a get_attr + # function, future calls to dep_attrs() will reflect this new data. + res_defns = stack_definition._resource_defns or {} + op_defns = stack_definition._output_defns or {} + + all_defns = itertools.chain(six.itervalues(res_defns), + six.itervalues(op_defns)) + for defn in all_defns: + if resource_name in defn.required_resource_names(): + defn._all_dep_attrs = None + def add_resource(stack_definition, resource_definition): """Insert the given resource definition into the stack definition. diff --git a/heat/tests/test_stack_collect_attributes.py b/heat/tests/test_stack_collect_attributes.py index 68e198f845..1757832633 100644 --- a/heat/tests/test_stack_collect_attributes.py +++ b/heat/tests/test_stack_collect_attributes.py @@ -224,18 +224,26 @@ class DepAttrsTest(common.HeatTestCase): super(DepAttrsTest, self).setUp() self.ctx = utils.dummy_context() - def test_dep_attrs(self): - parsed_tmpl = template_format.parse(self.tmpl) + self.parsed_tmpl = template_format.parse(self.tmpl) self.stack = stack.Stack(self.ctx, 'test_stack', - template.Template(parsed_tmpl)) + template.Template(self.parsed_tmpl)) + def test_dep_attrs(self): for res in six.itervalues(self.stack): definitions = (self.stack.defn.resource_definition(n) - for n in parsed_tmpl['resources']) + for n in self.parsed_tmpl['resources']) self.assertEqual(self.expected[res.name], set(itertools.chain.from_iterable( d.dep_attrs(res.name) for d in definitions))) + def test_all_dep_attrs(self): + for res in six.itervalues(self.stack): + definitions = (self.stack.defn.resource_definition(n) + for n in self.parsed_tmpl['resources']) + attrs = set(itertools.chain.from_iterable( + d.dep_attrs(res.name, load_all=True) for d in definitions)) + self.assertEqual(self.expected[res.name], attrs) + class ReferencedAttrsTest(common.HeatTestCase): def setUp(self):