Switch total_resources to use stack_count_total_resources
This change uses database queries to count the total number of resources in a nested stack tree instead of loading every stack and every resource in that stack. This should result in a significant improvement in memory use for stacks with many nested stacks and resources. Closes-Bug: #1455589 Change-Id: I8c2f1fa9114dbf68ad4a5f99dd3929b6958b5d7a
This commit is contained in:
parent
be41d08a44
commit
ca32acbece
@ -224,8 +224,9 @@ class StackResource(resource.Resource):
|
||||
return parsed_template
|
||||
|
||||
def _validate_nested_resources(self, templ):
|
||||
root_stack_id = self.stack.root_stack_id()
|
||||
total_resources = (len(templ[templ.RESOURCES]) +
|
||||
self.stack.root_stack.total_resources())
|
||||
self.stack.total_resources(root_stack_id))
|
||||
|
||||
if self.nested():
|
||||
# It's an update and these resources will be deleted
|
||||
|
@ -259,16 +259,7 @@ class Stack(collections.Mapping):
|
||||
def root_stack_id(self):
|
||||
if not self.owner_id:
|
||||
return self.id
|
||||
return stack_object.Stack.get_root_id(self.context, self.id)
|
||||
|
||||
@property
|
||||
def root_stack(self):
|
||||
'''
|
||||
Return the root stack if this is nested (otherwise return self).
|
||||
'''
|
||||
if (self.parent_resource and self.parent_resource.stack):
|
||||
return self.parent_resource.stack.root_stack
|
||||
return self
|
||||
return stack_object.Stack.get_root_id(self.context, self.owner_id)
|
||||
|
||||
def object_path_in_stack(self):
|
||||
'''
|
||||
@ -296,26 +287,14 @@ class Stack(collections.Mapping):
|
||||
return [(stckres.name if stckres else None,
|
||||
stck.name if stck else None) for stckres, stck in opis]
|
||||
|
||||
def total_resources(self):
|
||||
def total_resources(self, stack_id=None):
|
||||
'''
|
||||
Return the total number of resources in a stack, including nested
|
||||
stacks below.
|
||||
'''
|
||||
def total_nested(res):
|
||||
get_nested = getattr(res, 'nested', None)
|
||||
if callable(get_nested):
|
||||
try:
|
||||
nested_stack = get_nested()
|
||||
except exception.NotFound:
|
||||
# when an delete is underway, a nested stack can
|
||||
# disapear at any moment.
|
||||
return 0
|
||||
if nested_stack is not None:
|
||||
return nested_stack.total_resources()
|
||||
return 0
|
||||
|
||||
return len(self) + sum(total_nested(res)
|
||||
for res in six.itervalues(self))
|
||||
if not stack_id:
|
||||
stack_id = self.id
|
||||
return stack_object.Stack.count_total_resources(self.context, stack_id)
|
||||
|
||||
def _set_param_stackid(self):
|
||||
'''
|
||||
|
@ -24,6 +24,7 @@ from heat.engine import resource as res
|
||||
from heat.engine import service
|
||||
from heat.engine import stack
|
||||
from heat.engine import template as templatem
|
||||
from heat.objects import stack as stack_object
|
||||
from heat.openstack.common import threadgroup
|
||||
from heat.tests import common
|
||||
from heat.tests.engine import tools
|
||||
@ -206,7 +207,8 @@ class StackCreateTest(common.HeatTestCase):
|
||||
convergence=False,
|
||||
parent_resource=None)
|
||||
|
||||
def test_stack_create_total_resources_equals_max(self):
|
||||
@mock.patch.object(stack_object.Stack, 'count_total_resources')
|
||||
def test_stack_create_total_resources_equals_max(self, ctr):
|
||||
stack_name = 'stack_create_total_resources_equals_max'
|
||||
params = {}
|
||||
res._register_class('FakeResourceType', generic_rsrc.GenericResource)
|
||||
@ -221,6 +223,7 @@ class StackCreateTest(common.HeatTestCase):
|
||||
|
||||
template = templatem.Template(tpl)
|
||||
stk = stack.Stack(self.ctx, stack_name, template)
|
||||
ctr.return_value = 3
|
||||
|
||||
mock_tmpl = self.patchobject(templatem, 'Template', return_value=stk.t)
|
||||
mock_env = self.patchobject(environment, 'Environment',
|
||||
@ -242,7 +245,8 @@ class StackCreateTest(common.HeatTestCase):
|
||||
parent_resource=None)
|
||||
|
||||
self.assertEqual(stk.identifier(), result)
|
||||
self.assertEqual(3, stk.total_resources())
|
||||
root_stack_id = stk.root_stack_id()
|
||||
self.assertEqual(3, stk.total_resources(root_stack_id))
|
||||
self.man.thread_group_mgr.groups[stk.id].wait()
|
||||
stk.delete()
|
||||
|
||||
|
@ -1063,7 +1063,8 @@ class StackServiceAdoptUpdateDeleteTest(common.HeatTestCase):
|
||||
"('UPDATE', 'COMPLETE')",
|
||||
six.text_type(ex.exc_info[1]))
|
||||
|
||||
def test_stack_update_equals(self):
|
||||
@mock.patch.object(stack_object.Stack, 'count_total_resources')
|
||||
def test_stack_update_equals(self, ctr):
|
||||
stack_name = 'test_stack_update_equals_resource_limit'
|
||||
params = {}
|
||||
res._register_class('GenericResourceType',
|
||||
@ -1080,6 +1081,7 @@ class StackServiceAdoptUpdateDeleteTest(common.HeatTestCase):
|
||||
sid = old_stack.store()
|
||||
old_stack.set_stack_user_project_id('1234')
|
||||
s = stack_object.Stack.get_by_id(self.ctx, sid)
|
||||
ctr.return_value = 3
|
||||
|
||||
stack = parser.Stack(self.ctx, stack_name, template)
|
||||
|
||||
@ -1115,7 +1117,8 @@ class StackServiceAdoptUpdateDeleteTest(common.HeatTestCase):
|
||||
self.assertEqual(old_stack.identifier(), result)
|
||||
self.assertIsInstance(result, dict)
|
||||
self.assertTrue(result['stack_id'])
|
||||
self.assertEqual(3, old_stack.root_stack.total_resources())
|
||||
root_stack_id = old_stack.root_stack_id()
|
||||
self.assertEqual(3, old_stack.total_resources(root_stack_id))
|
||||
self.m.VerifyAll()
|
||||
|
||||
def test_stack_update_stack_id_equal(self):
|
||||
|
@ -84,7 +84,9 @@ Outputs:
|
||||
stack.store()
|
||||
return stack
|
||||
|
||||
def test_nested_stack_three_deep(self):
|
||||
@mock.patch.object(parser.Stack, 'root_stack_id')
|
||||
@mock.patch.object(parser.Stack, 'total_resources')
|
||||
def test_nested_stack_three_deep(self, tr, rsi):
|
||||
root_template = '''
|
||||
HeatTemplateFormatVersion: 2012-12-12
|
||||
Resources:
|
||||
@ -116,13 +118,19 @@ Resources:
|
||||
depth2_template,
|
||||
self.nested_template]
|
||||
|
||||
rsi.return_value = '1234'
|
||||
tr.return_value = 2
|
||||
|
||||
self.validate_stack(root_template)
|
||||
calls = [mock.call('https://server.test/depth1.template'),
|
||||
mock.call('https://server.test/depth2.template'),
|
||||
mock.call('https://server.test/depth3.template')]
|
||||
urlfetch.get.assert_has_calls(calls)
|
||||
tr.assert_called_with('1234')
|
||||
|
||||
def test_nested_stack_six_deep(self):
|
||||
@mock.patch.object(parser.Stack, 'root_stack_id')
|
||||
@mock.patch.object(parser.Stack, 'total_resources')
|
||||
def test_nested_stack_six_deep(self, tr, rsi):
|
||||
tmpl = '''
|
||||
HeatTemplateFormatVersion: 2012-12-12
|
||||
Resources:
|
||||
@ -150,6 +158,9 @@ Resources:
|
||||
depth5_template,
|
||||
self.nested_template]
|
||||
|
||||
rsi.return_value = '1234'
|
||||
tr.return_value = 5
|
||||
|
||||
t = template_format.parse(root_template)
|
||||
stack = self.parse_stack(t)
|
||||
res = self.assertRaises(exception.StackValidationFailed,
|
||||
@ -202,7 +213,9 @@ Resources:
|
||||
mock.call('https://server.test/depth4.template')]
|
||||
urlfetch.get.assert_has_calls(calls, any_order=True)
|
||||
|
||||
def test_nested_stack_infinite_recursion(self):
|
||||
@mock.patch.object(parser.Stack, 'root_stack_id')
|
||||
@mock.patch.object(parser.Stack, 'total_resources')
|
||||
def test_nested_stack_infinite_recursion(self, tr, rsi):
|
||||
tmpl = '''
|
||||
HeatTemplateFormatVersion: 2012-12-12
|
||||
Resources:
|
||||
@ -214,6 +227,8 @@ Resources:
|
||||
urlfetch.get.return_value = tmpl
|
||||
t = template_format.parse(tmpl)
|
||||
stack = self.parse_stack(t)
|
||||
rsi.return_value = '1234'
|
||||
tr.return_value = 2
|
||||
res = self.assertRaises(exception.StackValidationFailed,
|
||||
stack.validate)
|
||||
self.assertIn('Recursion depth exceeds', six.text_type(res))
|
||||
|
@ -172,39 +172,27 @@ class StackTest(common.HeatTestCase):
|
||||
def test_total_resources_empty(self):
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl,
|
||||
status_reason='flimflam')
|
||||
self.stack.store()
|
||||
self.assertEqual(0, self.stack.total_resources(self.stack.id))
|
||||
self.assertEqual(0, self.stack.total_resources())
|
||||
|
||||
def test_total_resources_generic(self):
|
||||
def test_total_resources_not_found(self):
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl,
|
||||
status_reason='flimflam')
|
||||
|
||||
self.assertEqual(0, self.stack.total_resources('1234'))
|
||||
|
||||
@mock.patch.object(db_api, 'stack_count_total_resources')
|
||||
def test_total_resources_generic(self, sctr):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
{'A': {'Type': 'GenericResourceType'}}}
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack',
|
||||
template.Template(tpl),
|
||||
status_reason='blarg')
|
||||
self.assertEqual(1, self.stack.total_resources())
|
||||
|
||||
def test_total_resources_nested_ok(self):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
{'A': {'Type': 'GenericResourceType'}}}
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack',
|
||||
template.Template(tpl),
|
||||
status_reason='blarg')
|
||||
|
||||
self.stack['A'].nested = mock.Mock()
|
||||
self.stack['A'].nested.return_value.total_resources.return_value = 3
|
||||
self.assertEqual(4, self.stack.total_resources())
|
||||
|
||||
def test_total_resources_nested_not_found(self):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
{'A': {'Type': 'GenericResourceType'}}}
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack',
|
||||
template.Template(tpl),
|
||||
status_reason='blarg')
|
||||
|
||||
self.stack['A'].nested = mock.Mock(
|
||||
side_effect=exception.NotFound('gone'))
|
||||
self.stack.store()
|
||||
sctr.return_value = 1
|
||||
self.assertEqual(1, self.stack.total_resources(self.stack.id))
|
||||
self.assertEqual(1, self.stack.total_resources())
|
||||
|
||||
def test_iter_resources(self):
|
||||
@ -264,42 +252,6 @@ class StackTest(common.HeatTestCase):
|
||||
# A cache supplied means we should never query the database.
|
||||
self.assertFalse(mock_drg.called)
|
||||
|
||||
def test_root_stack_no_parent(self):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
{'A': {'Type': 'GenericResourceType'}}}
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack',
|
||||
template.Template(tpl),
|
||||
status_reason='blarg')
|
||||
|
||||
self.assertEqual(self.stack, self.stack.root_stack)
|
||||
|
||||
def test_root_stack_parent_no_stack(self):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
{'A': {'Type': 'GenericResourceType'}}}
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack',
|
||||
template.Template(tpl),
|
||||
status_reason='blarg',
|
||||
parent_resource='parent')
|
||||
|
||||
parent_resource = mock.Mock()
|
||||
parent_resource.stack = None
|
||||
self.stack._parent_stack = dict(parent=parent_resource)
|
||||
self.assertEqual(self.stack, self.stack.root_stack)
|
||||
|
||||
def test_root_stack_with_parent(self):
|
||||
tpl = {'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources':
|
||||
{'A': {'Type': 'GenericResourceType'}}}
|
||||
stk = stack.Stack(self.ctx, 'test_stack', template.Template(tpl),
|
||||
status_reason='blarg', parent_resource='parent')
|
||||
|
||||
parent_resource = mock.Mock()
|
||||
parent_resource.stack.root_stack = 'test value'
|
||||
stk._parent_stack = dict(parent=parent_resource)
|
||||
self.assertEqual('test value', stk.root_stack)
|
||||
|
||||
def test_load_parent_resource(self):
|
||||
self.stack = stack.Stack(self.ctx, 'load_parent_resource', self.tmpl,
|
||||
parent_resource='parent')
|
||||
|
@ -434,7 +434,7 @@ class StackResourceTest(StackResourceBaseTest):
|
||||
'Resources': [1]}
|
||||
template = stack_resource.template.Template(tmpl)
|
||||
root_resources = mock.Mock(return_value=2)
|
||||
self.parent_resource.stack.root_stack.total_resources = root_resources
|
||||
self.parent_resource.stack.total_resources = root_resources
|
||||
|
||||
self.assertRaises(exception.RequestLimitExceeded,
|
||||
self.parent_resource._validate_nested_resources,
|
||||
@ -534,24 +534,21 @@ class StackResourceTest(StackResourceBaseTest):
|
||||
|
||||
class StackResourceLimitTest(StackResourceBaseTest):
|
||||
scenarios = [
|
||||
('1', dict(root=3, templ=4, nested=0, max=10, error=False)),
|
||||
('2', dict(root=3, templ=8, nested=0, max=10, error=True)),
|
||||
('3', dict(root=3, templ=8, nested=2, max=10, error=False)),
|
||||
('4', dict(root=3, templ=12, nested=2, max=10, error=True))]
|
||||
('3_4_0', dict(root=3, templ=4, nested=0, max=10, error=False)),
|
||||
('3_8_0', dict(root=3, templ=8, nested=0, max=10, error=True)),
|
||||
('3_8_2', dict(root=3, templ=8, nested=2, max=10, error=True)),
|
||||
('3_5_2', dict(root=3, templ=5, nested=2, max=10, error=False)),
|
||||
('3_6_2', dict(root=3, templ=6, nested=2, max=10, error=True)),
|
||||
('3_12_2', dict(root=3, templ=12, nested=2, max=10, error=True))]
|
||||
|
||||
def setUp(self):
|
||||
super(StackResourceLimitTest, self).setUp()
|
||||
self.res = self.parent_resource
|
||||
|
||||
def test_resource_limit(self):
|
||||
# mock nested resources
|
||||
nested = mock.MagicMock()
|
||||
nested.resources = range(self.nested)
|
||||
self.res.nested = mock.MagicMock(return_value=nested)
|
||||
|
||||
# mock root total_resources
|
||||
self.res.stack.root_stack.total_resources = mock.Mock(
|
||||
return_value=self.root)
|
||||
total_resources = self.root + self.nested
|
||||
parser.Stack.total_resources = mock.Mock(return_value=total_resources)
|
||||
|
||||
# setup the config max
|
||||
cfg.CONF.set_default('max_resources_per_stack', self.max)
|
||||
|
Loading…
Reference in New Issue
Block a user