Convergence: Allow creating lightweight stacks

Currently, we load all the resources from the database to resolve
template functions of dependent resources. In convergence, because
every worker will load it's own stack object, loading all the
resources for every resource lifecycle operation will be inefficient.

This patch allows creating lightweight stacks using a cache provided
which will never query the database and only depend on the template
and the cache provided. Function resolutions will now happen by querying
the values in the cache provided if it exists else None is returned.
The resultant lightweight stack will effectively be the stack attributes
loaded from the database and it's raw template with the dependent
resource's functions resolved which will serve as the input for
individual resource objects that will be worked upon.

blueprint convergence-lightweight-stack

Change-Id: I6dbaa7ee4e9d534c31823b4812efcb387c695a22
This commit is contained in:
Sirushti Murugesan 2015-04-10 21:23:04 +05:30
parent e4b959d75c
commit 0638030671
5 changed files with 147 additions and 16 deletions

View File

@ -198,6 +198,10 @@ class GetAtt(function.Function):
r = self._resource()
if (r.action in (r.CREATE, r.ADOPT, r.SUSPEND, r.RESUME, r.UPDATE)):
return r.FnGetAtt(attribute)
# NOTE(sirushtim): Add r.INIT to states above once convergence
# is the default.
elif r.stack.has_cache_data() and r.action == r.INIT:
return r.FnGetAtt(attribute)
else:
return None

View File

@ -180,9 +180,10 @@ class Resource(object):
self.replaced_by = None
self.current_template_id = None
resource = stack.db_resource_get(name)
if resource:
self._load_data(resource)
if not stack.has_cache_data():
resource = stack.db_resource_get(name)
if resource:
self._load_data(resource)
def rpc_client(self):
'''Return a client for making engine RPC calls.'''
@ -1091,6 +1092,9 @@ class Resource(object):
:results: the id or name of the resource.
'''
if self.stack.has_cache_data():
return self.stack.cache_data_resource_id(self.name)
if self.resource_id is not None:
return six.text_type(self.resource_id)
else:
@ -1111,13 +1115,18 @@ class Resource(object):
:param path: a list of path components to select from the attribute.
:returns: the attribute value.
'''
try:
attribute = self.attributes[key]
except KeyError:
raise exception.InvalidTemplateAttribute(resource=self.name,
key=key)
if self.stack.has_cache_data():
# Load from cache for lightweight resources.
attribute = self.stack.cache_data_resource_attribute(
self.name, key)
else:
return attributes.select_from_attribute(attribute, path)
try:
attribute = self.attributes[key]
except KeyError:
raise exception.InvalidTemplateAttribute(resource=self.name,
key=key)
return attributes.select_from_attribute(attribute, path)
def FnBase64(self, data):
'''

View File

@ -89,11 +89,16 @@ class Stack(collections.Mapping):
use_stored_context=False, username=None,
nested_depth=0, strict_validate=True, convergence=False,
current_traversal=None, tags=None, prev_raw_template_id=None,
current_deps=None):
current_deps=None, cache_data=None):
'''
Initialise from a context, name, Template object and (optionally)
Environment object. The database ID may also be initialised, if the
stack is already in the database.
Creating a stack with cache_data creates a lightweight stack which
will not load any resources from the database and resolve the
functions from the cache_data specified.
'''
def _validate_stack_name(name):
@ -135,6 +140,7 @@ class Stack(collections.Mapping):
self.tags = tags
self.prev_raw_template_id = prev_raw_template_id
self.current_deps = current_deps
self.cache_data = cache_data
if use_stored_context:
self.context = self.stored_context()
@ -339,8 +345,8 @@ class Stack(collections.Mapping):
return deps
@classmethod
def load(cls, context, stack_id=None, stack=None,
show_deleted=True, use_stored_context=False, force_reload=False):
def load(cls, context, stack_id=None, stack=None, show_deleted=True,
use_stored_context=False, force_reload=False, cache_data=None):
'''Retrieve a Stack from the database.'''
if stack is None:
stack = stack_object.Stack.get_by_id(
@ -356,7 +362,8 @@ class Stack(collections.Mapping):
stack.refresh()
return cls._from_db(context, stack,
use_stored_context=use_stored_context)
use_stored_context=use_stored_context,
cache_data=cache_data)
@classmethod
def load_all(cls, context, limit=None, marker=None, sort_keys=None,
@ -384,7 +391,7 @@ class Stack(collections.Mapping):
@classmethod
def _from_db(cls, context, stack, resolve_data=True,
use_stored_context=False):
use_stored_context=False, cache_data=None):
template = tmpl.Template.load(
context, stack.raw_template_id, stack.raw_template)
tags = None
@ -407,7 +414,7 @@ class Stack(collections.Mapping):
username=stack.username, convergence=stack.convergence,
current_traversal=stack.current_traversal, tags=tags,
prev_raw_template_id=stack.prev_raw_template_id,
current_deps=stack.current_deps)
current_deps=stack.current_deps, cache_data=cache_data)
def get_kwargs_for_cloning(self, keep_status=False, only_db=False):
"""Get common kwargs for calling Stack() for cloning.
@ -1572,3 +1579,16 @@ class Stack(collections.Mapping):
# of other resources, so ensure that attributes are re-calculated
for res in six.itervalues(self.resources):
res.attributes.reset_resolved_values()
def has_cache_data(self):
if self.cache_data is not None:
return True
return False
def cache_data_resource_id(self, resource_name):
return self.cache_data.get(resource_name, {}).get('id')
def cache_data_resource_attribute(self, resource_name, attribute_key):
return self.cache_data.get(
resource_name, {}).get('attributes', {}).get(attribute_key)

View File

@ -109,6 +109,7 @@ class NeutronTest(common.HeatTestCase):
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
stack = mock.MagicMock()
stack.has_cache_data = mock.Mock(return_value=False)
res = SomeNeutronResource('aresource', tmpl, stack)
mock_show_resource = mock.MagicMock()

View File

@ -232,6 +232,37 @@ class StackTest(common.HeatTestCase):
all_resources = list(self.stack.iter_resources(1))
self.assertEqual(5, len(all_resources))
@mock.patch.object(stack.Stack, 'db_resource_get')
def test_iter_resources_cached(self, mock_drg):
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
'Resources':
{'A': {'Type': 'GenericResourceType'},
'B': {'Type': 'GenericResourceType'}}}
self.stack = stack.Stack(self.ctx, 'test_stack',
template.Template(tpl),
status_reason='blarg',
cache_data={})
def get_more(nested_depth=0):
yield 'X'
yield 'Y'
yield 'Z'
self.stack['A'].nested = mock.MagicMock()
self.stack['A'].nested.return_value.iter_resources = mock.MagicMock(
side_effect=get_more)
resource_generator = self.stack.iter_resources()
self.assertIsNot(resource_generator, list)
first_level_resources = list(resource_generator)
self.assertEqual(2, len(first_level_resources))
all_resources = list(self.stack.iter_resources(1))
self.assertEqual(5, len(all_resources))
# A cache supplied means we should never query the database.
self.assertFalse(mock_drg.called)
def test_root_stack_no_parent(self):
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
'Resources':
@ -295,7 +326,7 @@ class StackTest(common.HeatTestCase):
current_traversal=None,
tags=mox.IgnoreArg(),
prev_raw_template_id=None,
current_deps=None)
current_deps=None, cache_data=None)
self.m.ReplayAll()
stack.Stack.load(self.ctx, stack_id=self.stack.id)
@ -1853,6 +1884,72 @@ class StackTest(common.HeatTestCase):
self.assertEqual(
'foo', self.stack.resources['A'].properties['a_string'])
@mock.patch.object(stack.Stack, 'db_resource_get')
def test_lightweight_stack_getatt(self, mock_drg):
tmpl = template.Template({
'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
'foo': {'Type': 'GenericResourceType'},
'bar': {
'Type': 'ResourceWithPropsType',
'Properties': {
'Foo': {'Fn::GetAtt': ['foo', 'bar']},
}
}
}
})
cache_data = {'foo': {'attributes': {'bar': 'baz'}}}
tmpl_stack = stack.Stack(self.ctx, 'test', tmpl)
tmpl_stack.store()
lightweight_stack = stack.Stack.load(self.ctx, stack_id=tmpl_stack.id,
cache_data=cache_data)
# Check if the property has the appropriate resolved value.
cached_property = lightweight_stack['bar'].properties['Foo']
self.assertEqual(cached_property, 'baz')
# Make sure FnGetAtt returns the cached value.
attr_value = lightweight_stack['foo'].FnGetAtt('bar')
self.assertEqual('baz', attr_value)
# Make sure calls are not made to the database to retrieve the
# resource state.
self.assertFalse(mock_drg.called)
@mock.patch.object(stack.Stack, 'db_resource_get')
def test_lightweight_stack_getrefid(self, mock_drg):
tmpl = template.Template({
'HeatTemplateFormatVersion': '2012-12-12',
'Resources': {
'foo': {'Type': 'GenericResourceType'},
'bar': {
'Type': 'ResourceWithPropsType',
'Properties': {
'Foo': {'Ref': 'foo'},
}
}
}
})
cache_data = {'foo': {'id': 'physical-resource-id'}}
tmpl_stack = stack.Stack(self.ctx, 'test', tmpl)
tmpl_stack.store()
lightweight_stack = stack.Stack.load(self.ctx, stack_id=tmpl_stack.id,
cache_data=cache_data)
# Check if the property has the appropriate resolved value.
cached_property = lightweight_stack['bar'].properties['Foo']
self.assertEqual(cached_property, 'physical-resource-id')
# Make sure FnGetRefId returns the cached value.
resource_id = lightweight_stack['foo'].FnGetRefId()
self.assertEqual('physical-resource-id', resource_id)
# Make sure calls are not made to the database to retrieve the
# resource state.
self.assertFalse(mock_drg.called)
class StackKwargsForCloningTest(common.HeatTestCase):
scenarios = [