Convergence: Allow creating lightweight stacks
Currently, we load all the resources from the database to resolve template functions of dependent resources. In convergence, because every worker will load it's own stack object, loading all the resources for every resource lifecycle operation will be inefficient. This patch allows creating lightweight stacks using a cache provided which will never query the database and only depend on the template and the cache provided. Function resolutions will now happen by querying the values in the cache provided if it exists else None is returned. The resultant lightweight stack will effectively be the stack attributes loaded from the database and it's raw template with the dependent resource's functions resolved which will serve as the input for individual resource objects that will be worked upon. blueprint convergence-lightweight-stack Change-Id: I6dbaa7ee4e9d534c31823b4812efcb387c695a22
This commit is contained in:
parent
e4b959d75c
commit
0638030671
@ -198,6 +198,10 @@ class GetAtt(function.Function):
|
||||
r = self._resource()
|
||||
if (r.action in (r.CREATE, r.ADOPT, r.SUSPEND, r.RESUME, r.UPDATE)):
|
||||
return r.FnGetAtt(attribute)
|
||||
# NOTE(sirushtim): Add r.INIT to states above once convergence
|
||||
# is the default.
|
||||
elif r.stack.has_cache_data() and r.action == r.INIT:
|
||||
return r.FnGetAtt(attribute)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
@ -180,9 +180,10 @@ class Resource(object):
|
||||
self.replaced_by = None
|
||||
self.current_template_id = None
|
||||
|
||||
resource = stack.db_resource_get(name)
|
||||
if resource:
|
||||
self._load_data(resource)
|
||||
if not stack.has_cache_data():
|
||||
resource = stack.db_resource_get(name)
|
||||
if resource:
|
||||
self._load_data(resource)
|
||||
|
||||
def rpc_client(self):
|
||||
'''Return a client for making engine RPC calls.'''
|
||||
@ -1091,6 +1092,9 @@ class Resource(object):
|
||||
|
||||
:results: the id or name of the resource.
|
||||
'''
|
||||
if self.stack.has_cache_data():
|
||||
return self.stack.cache_data_resource_id(self.name)
|
||||
|
||||
if self.resource_id is not None:
|
||||
return six.text_type(self.resource_id)
|
||||
else:
|
||||
@ -1111,13 +1115,18 @@ class Resource(object):
|
||||
:param path: a list of path components to select from the attribute.
|
||||
:returns: the attribute value.
|
||||
'''
|
||||
try:
|
||||
attribute = self.attributes[key]
|
||||
except KeyError:
|
||||
raise exception.InvalidTemplateAttribute(resource=self.name,
|
||||
key=key)
|
||||
if self.stack.has_cache_data():
|
||||
# Load from cache for lightweight resources.
|
||||
attribute = self.stack.cache_data_resource_attribute(
|
||||
self.name, key)
|
||||
else:
|
||||
return attributes.select_from_attribute(attribute, path)
|
||||
try:
|
||||
attribute = self.attributes[key]
|
||||
except KeyError:
|
||||
raise exception.InvalidTemplateAttribute(resource=self.name,
|
||||
key=key)
|
||||
|
||||
return attributes.select_from_attribute(attribute, path)
|
||||
|
||||
def FnBase64(self, data):
|
||||
'''
|
||||
|
@ -89,11 +89,16 @@ class Stack(collections.Mapping):
|
||||
use_stored_context=False, username=None,
|
||||
nested_depth=0, strict_validate=True, convergence=False,
|
||||
current_traversal=None, tags=None, prev_raw_template_id=None,
|
||||
current_deps=None):
|
||||
current_deps=None, cache_data=None):
|
||||
|
||||
'''
|
||||
Initialise from a context, name, Template object and (optionally)
|
||||
Environment object. The database ID may also be initialised, if the
|
||||
stack is already in the database.
|
||||
|
||||
Creating a stack with cache_data creates a lightweight stack which
|
||||
will not load any resources from the database and resolve the
|
||||
functions from the cache_data specified.
|
||||
'''
|
||||
|
||||
def _validate_stack_name(name):
|
||||
@ -135,6 +140,7 @@ class Stack(collections.Mapping):
|
||||
self.tags = tags
|
||||
self.prev_raw_template_id = prev_raw_template_id
|
||||
self.current_deps = current_deps
|
||||
self.cache_data = cache_data
|
||||
|
||||
if use_stored_context:
|
||||
self.context = self.stored_context()
|
||||
@ -339,8 +345,8 @@ class Stack(collections.Mapping):
|
||||
return deps
|
||||
|
||||
@classmethod
|
||||
def load(cls, context, stack_id=None, stack=None,
|
||||
show_deleted=True, use_stored_context=False, force_reload=False):
|
||||
def load(cls, context, stack_id=None, stack=None, show_deleted=True,
|
||||
use_stored_context=False, force_reload=False, cache_data=None):
|
||||
'''Retrieve a Stack from the database.'''
|
||||
if stack is None:
|
||||
stack = stack_object.Stack.get_by_id(
|
||||
@ -356,7 +362,8 @@ class Stack(collections.Mapping):
|
||||
stack.refresh()
|
||||
|
||||
return cls._from_db(context, stack,
|
||||
use_stored_context=use_stored_context)
|
||||
use_stored_context=use_stored_context,
|
||||
cache_data=cache_data)
|
||||
|
||||
@classmethod
|
||||
def load_all(cls, context, limit=None, marker=None, sort_keys=None,
|
||||
@ -384,7 +391,7 @@ class Stack(collections.Mapping):
|
||||
|
||||
@classmethod
|
||||
def _from_db(cls, context, stack, resolve_data=True,
|
||||
use_stored_context=False):
|
||||
use_stored_context=False, cache_data=None):
|
||||
template = tmpl.Template.load(
|
||||
context, stack.raw_template_id, stack.raw_template)
|
||||
tags = None
|
||||
@ -407,7 +414,7 @@ class Stack(collections.Mapping):
|
||||
username=stack.username, convergence=stack.convergence,
|
||||
current_traversal=stack.current_traversal, tags=tags,
|
||||
prev_raw_template_id=stack.prev_raw_template_id,
|
||||
current_deps=stack.current_deps)
|
||||
current_deps=stack.current_deps, cache_data=cache_data)
|
||||
|
||||
def get_kwargs_for_cloning(self, keep_status=False, only_db=False):
|
||||
"""Get common kwargs for calling Stack() for cloning.
|
||||
@ -1572,3 +1579,16 @@ class Stack(collections.Mapping):
|
||||
# of other resources, so ensure that attributes are re-calculated
|
||||
for res in six.itervalues(self.resources):
|
||||
res.attributes.reset_resolved_values()
|
||||
|
||||
def has_cache_data(self):
|
||||
if self.cache_data is not None:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def cache_data_resource_id(self, resource_name):
|
||||
return self.cache_data.get(resource_name, {}).get('id')
|
||||
|
||||
def cache_data_resource_attribute(self, resource_name, attribute_key):
|
||||
return self.cache_data.get(
|
||||
resource_name, {}).get('attributes', {}).get(attribute_key)
|
||||
|
@ -109,6 +109,7 @@ class NeutronTest(common.HeatTestCase):
|
||||
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
|
||||
stack = mock.MagicMock()
|
||||
stack.has_cache_data = mock.Mock(return_value=False)
|
||||
res = SomeNeutronResource('aresource', tmpl, stack)
|
||||
|
||||
mock_show_resource = mock.MagicMock()
|
||||
|
@ -232,6 +232,37 @@ class StackTest(common.HeatTestCase):
|
||||
all_resources = list(self.stack.iter_resources(1))
|
||||
self.assertEqual(5, len(all_resources))
|
||||
|
||||
@mock.patch.object(stack.Stack, 'db_resource_get')
|
||||
def test_iter_resources_cached(self, mock_drg):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
{'A': {'Type': 'GenericResourceType'},
|
||||
'B': {'Type': 'GenericResourceType'}}}
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack',
|
||||
template.Template(tpl),
|
||||
status_reason='blarg',
|
||||
cache_data={})
|
||||
|
||||
def get_more(nested_depth=0):
|
||||
yield 'X'
|
||||
yield 'Y'
|
||||
yield 'Z'
|
||||
|
||||
self.stack['A'].nested = mock.MagicMock()
|
||||
self.stack['A'].nested.return_value.iter_resources = mock.MagicMock(
|
||||
side_effect=get_more)
|
||||
|
||||
resource_generator = self.stack.iter_resources()
|
||||
self.assertIsNot(resource_generator, list)
|
||||
|
||||
first_level_resources = list(resource_generator)
|
||||
self.assertEqual(2, len(first_level_resources))
|
||||
all_resources = list(self.stack.iter_resources(1))
|
||||
self.assertEqual(5, len(all_resources))
|
||||
|
||||
# A cache supplied means we should never query the database.
|
||||
self.assertFalse(mock_drg.called)
|
||||
|
||||
def test_root_stack_no_parent(self):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
@ -295,7 +326,7 @@ class StackTest(common.HeatTestCase):
|
||||
current_traversal=None,
|
||||
tags=mox.IgnoreArg(),
|
||||
prev_raw_template_id=None,
|
||||
current_deps=None)
|
||||
current_deps=None, cache_data=None)
|
||||
|
||||
self.m.ReplayAll()
|
||||
stack.Stack.load(self.ctx, stack_id=self.stack.id)
|
||||
@ -1853,6 +1884,72 @@ class StackTest(common.HeatTestCase):
|
||||
self.assertEqual(
|
||||
'foo', self.stack.resources['A'].properties['a_string'])
|
||||
|
||||
@mock.patch.object(stack.Stack, 'db_resource_get')
|
||||
def test_lightweight_stack_getatt(self, mock_drg):
|
||||
tmpl = template.Template({
|
||||
'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources': {
|
||||
'foo': {'Type': 'GenericResourceType'},
|
||||
'bar': {
|
||||
'Type': 'ResourceWithPropsType',
|
||||
'Properties': {
|
||||
'Foo': {'Fn::GetAtt': ['foo', 'bar']},
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
cache_data = {'foo': {'attributes': {'bar': 'baz'}}}
|
||||
tmpl_stack = stack.Stack(self.ctx, 'test', tmpl)
|
||||
tmpl_stack.store()
|
||||
lightweight_stack = stack.Stack.load(self.ctx, stack_id=tmpl_stack.id,
|
||||
cache_data=cache_data)
|
||||
|
||||
# Check if the property has the appropriate resolved value.
|
||||
cached_property = lightweight_stack['bar'].properties['Foo']
|
||||
self.assertEqual(cached_property, 'baz')
|
||||
|
||||
# Make sure FnGetAtt returns the cached value.
|
||||
attr_value = lightweight_stack['foo'].FnGetAtt('bar')
|
||||
self.assertEqual('baz', attr_value)
|
||||
|
||||
# Make sure calls are not made to the database to retrieve the
|
||||
# resource state.
|
||||
self.assertFalse(mock_drg.called)
|
||||
|
||||
@mock.patch.object(stack.Stack, 'db_resource_get')
|
||||
def test_lightweight_stack_getrefid(self, mock_drg):
|
||||
tmpl = template.Template({
|
||||
'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources': {
|
||||
'foo': {'Type': 'GenericResourceType'},
|
||||
'bar': {
|
||||
'Type': 'ResourceWithPropsType',
|
||||
'Properties': {
|
||||
'Foo': {'Ref': 'foo'},
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
cache_data = {'foo': {'id': 'physical-resource-id'}}
|
||||
tmpl_stack = stack.Stack(self.ctx, 'test', tmpl)
|
||||
tmpl_stack.store()
|
||||
lightweight_stack = stack.Stack.load(self.ctx, stack_id=tmpl_stack.id,
|
||||
cache_data=cache_data)
|
||||
|
||||
# Check if the property has the appropriate resolved value.
|
||||
cached_property = lightweight_stack['bar'].properties['Foo']
|
||||
self.assertEqual(cached_property, 'physical-resource-id')
|
||||
|
||||
# Make sure FnGetRefId returns the cached value.
|
||||
resource_id = lightweight_stack['foo'].FnGetRefId()
|
||||
self.assertEqual('physical-resource-id', resource_id)
|
||||
|
||||
# Make sure calls are not made to the database to retrieve the
|
||||
# resource state.
|
||||
self.assertFalse(mock_drg.called)
|
||||
|
||||
|
||||
class StackKwargsForCloningTest(common.HeatTestCase):
|
||||
scenarios = [
|
||||
|
Loading…
Reference in New Issue
Block a user