Cache dep_attrs for all resources in definitions
When the dep_attrs function was written, it was used only in convergence after checking a single resource. However, now we also use it to generate data for the ResourceProxy objects, which is often done for all resources simultaneously. That means doing O(n^2) dep_attrs() calls, which can really slow things down when there is a large number of resources with complex properties (or metadata). This change adds an all_dep_attrs() call, which runs as fast as dep_attrs() on everything except get_attr functions and their arguments, but only needs to be called once instead of once per resource. (The get_attr function can in future override the default implementation of all_dep_attrs() to be as efficient as dep_attrs() as well.) The resulting data is cached in the ResourceDefinition or OutputDefinition so that subsequent calls to their get_attr() methods with different (or the same) resource names will use the existing data. Change-Id: If95f4c04b841519ce3d7492211f2696588c0ed48 Partially-Implements: blueprint stack-definition Closes-Bug: #1684272
This commit is contained in:
parent
b0916ad5bb
commit
3c13cb82a6
|
@ -80,6 +80,32 @@ class Function(object):
|
||||||
"""
|
"""
|
||||||
return dep_attrs(self.args, resource_name)
|
return dep_attrs(self.args, resource_name)
|
||||||
|
|
||||||
|
def all_dep_attrs(self):
|
||||||
|
"""Return resource, attribute name pairs of all attributes referenced.
|
||||||
|
|
||||||
|
Return an iterator over the resource name, attribute name tuples of
|
||||||
|
all attributes that this function references.
|
||||||
|
|
||||||
|
The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to
|
||||||
|
indicate that all attributes of the resource are required.
|
||||||
|
|
||||||
|
By default this calls the dep_attrs() method, but subclasses can
|
||||||
|
override to provide a more efficient implementation.
|
||||||
|
"""
|
||||||
|
# If we are using the default dep_attrs method then it will only
|
||||||
|
# return data from the args anyway
|
||||||
|
if type(self).dep_attrs == Function.dep_attrs:
|
||||||
|
return all_dep_attrs(self.args)
|
||||||
|
|
||||||
|
def res_dep_attrs(resource_name):
|
||||||
|
return six.moves.zip(itertools.repeat(resource_name),
|
||||||
|
self.dep_attrs(resource_name))
|
||||||
|
|
||||||
|
resource_names = self.stack.enabled_rsrc_names()
|
||||||
|
|
||||||
|
return itertools.chain.from_iterable(six.moves.map(res_dep_attrs,
|
||||||
|
resource_names))
|
||||||
|
|
||||||
def __reduce__(self):
|
def __reduce__(self):
|
||||||
"""Return a representation of the function suitable for pickling.
|
"""Return a representation of the function suitable for pickling.
|
||||||
|
|
||||||
|
@ -189,6 +215,25 @@ class Macro(Function):
|
||||||
"""
|
"""
|
||||||
return dep_attrs(self.parsed, resource_name)
|
return dep_attrs(self.parsed, resource_name)
|
||||||
|
|
||||||
|
def all_dep_attrs(self):
|
||||||
|
"""Return resource, attribute name pairs of all attributes referenced.
|
||||||
|
|
||||||
|
Return an iterator over the resource name, attribute name tuples of
|
||||||
|
all attributes that this function references.
|
||||||
|
|
||||||
|
The special value heat.engine.attributes.ALL_ATTRIBUTES may be used to
|
||||||
|
indicate that all attributes of the resource are required.
|
||||||
|
|
||||||
|
By default this calls the dep_attrs() method, but subclasses can
|
||||||
|
override to provide a more efficient implementation.
|
||||||
|
"""
|
||||||
|
# If we are using the default dep_attrs method then it will only
|
||||||
|
# return data from the transformed parsed args anyway
|
||||||
|
if type(self).dep_attrs == Macro.dep_attrs:
|
||||||
|
return all_dep_attrs(self.parsed)
|
||||||
|
|
||||||
|
return super(Macro, self).all_dep_attrs()
|
||||||
|
|
||||||
def __reduce__(self):
|
def __reduce__(self):
|
||||||
"""Return a representation of the macro result suitable for pickling.
|
"""Return a representation of the macro result suitable for pickling.
|
||||||
|
|
||||||
|
@ -299,6 +344,29 @@ def dep_attrs(snippet, resource_name):
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def all_dep_attrs(snippet):
|
||||||
|
"""Iterator over resource, attribute name pairs referenced in a snippet.
|
||||||
|
|
||||||
|
The snippet should be already parsed to insert Function objects where
|
||||||
|
appropriate.
|
||||||
|
|
||||||
|
:returns: an iterator over the resource name, attribute name tuples of all
|
||||||
|
attributes that are referenced in the template snippet.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if isinstance(snippet, Function):
|
||||||
|
return snippet.all_dep_attrs()
|
||||||
|
|
||||||
|
elif isinstance(snippet, collections.Mapping):
|
||||||
|
res_attrs = (all_dep_attrs(value) for value in snippet.values())
|
||||||
|
return itertools.chain.from_iterable(res_attrs)
|
||||||
|
elif (not isinstance(snippet, six.string_types) and
|
||||||
|
isinstance(snippet, collections.Iterable)):
|
||||||
|
res_attrs = (all_dep_attrs(value) for value in snippet)
|
||||||
|
return itertools.chain.from_iterable(res_attrs)
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
class Invalid(Function):
|
class Invalid(Function):
|
||||||
"""A function for checking condition functions and to force failures.
|
"""A function for checking condition functions and to force failures.
|
||||||
|
|
||||||
|
|
|
@ -11,6 +11,7 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
import collections
|
||||||
import copy
|
import copy
|
||||||
import six
|
import six
|
||||||
|
|
||||||
|
@ -27,6 +28,7 @@ class OutputDefinition(object):
|
||||||
self._resolved_value = None
|
self._resolved_value = None
|
||||||
self._description = description
|
self._description = description
|
||||||
self._deps = None
|
self._deps = None
|
||||||
|
self._all_dep_attrs = None
|
||||||
|
|
||||||
def validate(self, path=''):
|
def validate(self, path=''):
|
||||||
"""Validate the output value without resolving it."""
|
"""Validate the output value without resolving it."""
|
||||||
|
@ -44,12 +46,21 @@ class OutputDefinition(object):
|
||||||
self._deps = set()
|
self._deps = set()
|
||||||
return self._deps
|
return self._deps
|
||||||
|
|
||||||
def dep_attrs(self, resource_name):
|
def dep_attrs(self, resource_name, load_all=False):
|
||||||
"""Iterate over attributes of a given resource that this references.
|
"""Iterate over attributes of a given resource that this references.
|
||||||
|
|
||||||
Return an iterator over dependent attributes for specified
|
Return an iterator over dependent attributes for specified
|
||||||
resource_name in the output's value field.
|
resource_name in the output's value field.
|
||||||
"""
|
"""
|
||||||
|
if self._all_dep_attrs is None and load_all:
|
||||||
|
attr_map = collections.defaultdict(set)
|
||||||
|
for r, a in function.all_dep_attrs(self._value):
|
||||||
|
attr_map[r].add(a)
|
||||||
|
self._all_dep_attrs = attr_map
|
||||||
|
|
||||||
|
if self._all_dep_attrs is not None:
|
||||||
|
return iter(self._all_dep_attrs.get(resource_name, []))
|
||||||
|
|
||||||
return function.dep_attrs(self._value, resource_name)
|
return function.dep_attrs(self._value, resource_name)
|
||||||
|
|
||||||
def get_value(self):
|
def get_value(self):
|
||||||
|
|
|
@ -948,7 +948,8 @@ class Resource(status.ResourceStatus):
|
||||||
self.attributes.reset_resolved_values()
|
self.attributes.reset_resolved_values()
|
||||||
|
|
||||||
def referenced_attrs(self, stk_defn=None,
|
def referenced_attrs(self, stk_defn=None,
|
||||||
in_resources=True, in_outputs=True):
|
in_resources=True, in_outputs=True,
|
||||||
|
load_all=False):
|
||||||
"""Return the set of all attributes referenced in the template.
|
"""Return the set of all attributes referenced in the template.
|
||||||
|
|
||||||
This enables the resource to calculate which of its attributes will
|
This enables the resource to calculate which of its attributes will
|
||||||
|
@ -966,7 +967,8 @@ class Resource(status.ResourceStatus):
|
||||||
stk_defn = self.stack.defn
|
stk_defn = self.stack.defn
|
||||||
|
|
||||||
def get_dep_attrs(source):
|
def get_dep_attrs(source):
|
||||||
return set(itertools.chain.from_iterable(s.dep_attrs(self.name)
|
return set(itertools.chain.from_iterable(s.dep_attrs(self.name,
|
||||||
|
load_all)
|
||||||
for s in source))
|
for s in source))
|
||||||
|
|
||||||
refd_attrs = set()
|
refd_attrs = set()
|
||||||
|
@ -1030,13 +1032,16 @@ class Resource(status.ResourceStatus):
|
||||||
except exception.InvalidTemplateAttribute as ita:
|
except exception.InvalidTemplateAttribute as ita:
|
||||||
LOG.info('%s', ita)
|
LOG.info('%s', ita)
|
||||||
|
|
||||||
|
load_all = not self.stack.in_convergence_check
|
||||||
dep_attrs = self.referenced_attrs(stk_defn,
|
dep_attrs = self.referenced_attrs(stk_defn,
|
||||||
in_resources=for_resources,
|
in_resources=for_resources,
|
||||||
in_outputs=for_outputs)
|
in_outputs=for_outputs,
|
||||||
|
load_all=load_all)
|
||||||
|
|
||||||
# Ensure all attributes referenced in outputs get cached
|
# Ensure all attributes referenced in outputs get cached
|
||||||
if for_outputs is False and self.stack.convergence:
|
if for_outputs is False and self.stack.convergence:
|
||||||
out_attrs = self.referenced_attrs(stk_defn, in_resources=False)
|
out_attrs = self.referenced_attrs(stk_defn, in_resources=False,
|
||||||
|
load_all=load_all)
|
||||||
for e in get_attrs(out_attrs - dep_attrs, cacheable_only=True):
|
for e in get_attrs(out_attrs - dep_attrs, cacheable_only=True):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
|
@ -100,6 +100,7 @@ class ResourceDefinition(object):
|
||||||
self._hash = hash(self.resource_type)
|
self._hash = hash(self.resource_type)
|
||||||
self._rendering = None
|
self._rendering = None
|
||||||
self._dep_names = None
|
self._dep_names = None
|
||||||
|
self._all_dep_attrs = None
|
||||||
|
|
||||||
assert isinstance(self.description, six.string_types)
|
assert isinstance(self.description, six.string_types)
|
||||||
|
|
||||||
|
@ -192,12 +193,23 @@ class ResourceDefinition(object):
|
||||||
external_id=reparse_snippet(self._external_id),
|
external_id=reparse_snippet(self._external_id),
|
||||||
condition=self._condition)
|
condition=self._condition)
|
||||||
|
|
||||||
def dep_attrs(self, resource_name):
|
def dep_attrs(self, resource_name, load_all=False):
|
||||||
"""Iterate over attributes of a given resource that this references.
|
"""Iterate over attributes of a given resource that this references.
|
||||||
|
|
||||||
Return an iterator over dependent attributes for specified
|
Return an iterator over dependent attributes for specified
|
||||||
resource_name in resources' properties and metadata fields.
|
resource_name in resources' properties and metadata fields.
|
||||||
"""
|
"""
|
||||||
|
if self._all_dep_attrs is None and load_all:
|
||||||
|
attr_map = collections.defaultdict(set)
|
||||||
|
atts = itertools.chain(function.all_dep_attrs(self._properties),
|
||||||
|
function.all_dep_attrs(self._metadata))
|
||||||
|
for res_name, att_name in atts:
|
||||||
|
attr_map[res_name].add(att_name)
|
||||||
|
self._all_dep_attrs = attr_map
|
||||||
|
|
||||||
|
if self._all_dep_attrs is not None:
|
||||||
|
return self._all_dep_attrs[resource_name]
|
||||||
|
|
||||||
return itertools.chain(function.dep_attrs(self._properties,
|
return itertools.chain(function.dep_attrs(self._properties,
|
||||||
resource_name),
|
resource_name),
|
||||||
function.dep_attrs(self._metadata,
|
function.dep_attrs(self._metadata,
|
||||||
|
|
|
@ -11,6 +11,7 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
import itertools
|
||||||
import six
|
import six
|
||||||
|
|
||||||
from heat.common import exception
|
from heat.common import exception
|
||||||
|
@ -245,6 +246,19 @@ def update_resource_data(stack_definition, resource_name, resource_data):
|
||||||
stack_definition._resource_data[resource_name] = resource_data
|
stack_definition._resource_data[resource_name] = resource_data
|
||||||
stack_definition._resources.pop(resource_name, None)
|
stack_definition._resources.pop(resource_name, None)
|
||||||
|
|
||||||
|
# Clear the cached dep_attrs for any resource or output that directly
|
||||||
|
# depends on the resource whose data we are updating. This ensures that if
|
||||||
|
# any of the data we just updated is referenced in the path of a get_attr
|
||||||
|
# function, future calls to dep_attrs() will reflect this new data.
|
||||||
|
res_defns = stack_definition._resource_defns or {}
|
||||||
|
op_defns = stack_definition._output_defns or {}
|
||||||
|
|
||||||
|
all_defns = itertools.chain(six.itervalues(res_defns),
|
||||||
|
six.itervalues(op_defns))
|
||||||
|
for defn in all_defns:
|
||||||
|
if resource_name in defn.required_resource_names():
|
||||||
|
defn._all_dep_attrs = None
|
||||||
|
|
||||||
|
|
||||||
def add_resource(stack_definition, resource_definition):
|
def add_resource(stack_definition, resource_definition):
|
||||||
"""Insert the given resource definition into the stack definition.
|
"""Insert the given resource definition into the stack definition.
|
||||||
|
|
|
@ -224,18 +224,26 @@ class DepAttrsTest(common.HeatTestCase):
|
||||||
super(DepAttrsTest, self).setUp()
|
super(DepAttrsTest, self).setUp()
|
||||||
self.ctx = utils.dummy_context()
|
self.ctx = utils.dummy_context()
|
||||||
|
|
||||||
def test_dep_attrs(self):
|
self.parsed_tmpl = template_format.parse(self.tmpl)
|
||||||
parsed_tmpl = template_format.parse(self.tmpl)
|
|
||||||
self.stack = stack.Stack(self.ctx, 'test_stack',
|
self.stack = stack.Stack(self.ctx, 'test_stack',
|
||||||
template.Template(parsed_tmpl))
|
template.Template(self.parsed_tmpl))
|
||||||
|
|
||||||
|
def test_dep_attrs(self):
|
||||||
for res in six.itervalues(self.stack):
|
for res in six.itervalues(self.stack):
|
||||||
definitions = (self.stack.defn.resource_definition(n)
|
definitions = (self.stack.defn.resource_definition(n)
|
||||||
for n in parsed_tmpl['resources'])
|
for n in self.parsed_tmpl['resources'])
|
||||||
self.assertEqual(self.expected[res.name],
|
self.assertEqual(self.expected[res.name],
|
||||||
set(itertools.chain.from_iterable(
|
set(itertools.chain.from_iterable(
|
||||||
d.dep_attrs(res.name) for d in definitions)))
|
d.dep_attrs(res.name) for d in definitions)))
|
||||||
|
|
||||||
|
def test_all_dep_attrs(self):
|
||||||
|
for res in six.itervalues(self.stack):
|
||||||
|
definitions = (self.stack.defn.resource_definition(n)
|
||||||
|
for n in self.parsed_tmpl['resources'])
|
||||||
|
attrs = set(itertools.chain.from_iterable(
|
||||||
|
d.dep_attrs(res.name, load_all=True) for d in definitions))
|
||||||
|
self.assertEqual(self.expected[res.name], attrs)
|
||||||
|
|
||||||
|
|
||||||
class ReferencedAttrsTest(common.HeatTestCase):
|
class ReferencedAttrsTest(common.HeatTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
|
Loading…
Reference in New Issue