Switch total_resources to use stack_count_total_resources

This change uses database queries to count the total number of
resources in a nested stack tree instead of loading every stack
and every resource in that stack.

This should result in a significant improvement in memory use
for stacks with many nested stacks and resources.

Closes-Bug: #1455589
Change-Id: I8c2f1fa9114dbf68ad4a5f99dd3929b6958b5d7a

heat/engine/resources/stack_resource.py

@@ -224,8 +224,9 @@ class StackResource(resource.Resource):
         return parsed_template
 
     def _validate_nested_resources(self, templ):
+        root_stack_id = self.stack.root_stack_id()
         total_resources = (len(templ[templ.RESOURCES]) +
-                           self.stack.root_stack.total_resources())
+                           self.stack.total_resources(root_stack_id))
 
         if self.nested():
             # It's an update and these resources will be deleted

heat/engine/stack.py

@@ -259,16 +259,7 @@ class Stack(collections.Mapping):
     def root_stack_id(self):
         if not self.owner_id:
             return self.id
-        return stack_object.Stack.get_root_id(self.context, self.id)
+        return stack_object.Stack.get_root_id(self.context, self.owner_id)
-
-    @property
-    def root_stack(self):
-        '''
-        Return the root stack if this is nested (otherwise return self).
-        '''
-        if (self.parent_resource and self.parent_resource.stack):
-            return self.parent_resource.stack.root_stack
-        return self
 
     def object_path_in_stack(self):
         '''
@@ -296,26 +287,14 @@ class Stack(collections.Mapping):
         return [(stckres.name if stckres else None,
                  stck.name if stck else None) for stckres, stck in opis]
 
-    def total_resources(self):
+    def total_resources(self, stack_id=None):
         '''
         Return the total number of resources in a stack, including nested
         stacks below.
         '''
-        def total_nested(res):
+        if not stack_id:
-            get_nested = getattr(res, 'nested', None)
+            stack_id = self.id
-            if callable(get_nested):
+        return stack_object.Stack.count_total_resources(self.context, stack_id)
-                try:
-                    nested_stack = get_nested()
-                except exception.NotFound:
-                    # when an delete is underway, a nested stack can
-                    # disapear at any moment.
-                    return 0
-                if nested_stack is not None:
-                    return nested_stack.total_resources()
-            return 0
-
-        return len(self) + sum(total_nested(res)
-                               for res in six.itervalues(self))
 
     def _set_param_stackid(self):
         '''

heat/tests/engine/test_stack_create.py

@@ -24,6 +24,7 @@ from heat.engine import resource as res
 from heat.engine import service
 from heat.engine import stack
 from heat.engine import template as templatem
+from heat.objects import stack as stack_object
 from heat.openstack.common import threadgroup
 from heat.tests import common
 from heat.tests.engine import tools
@@ -206,7 +207,8 @@ class StackCreateTest(common.HeatTestCase):
                                            convergence=False,
                                            parent_resource=None)
 
-    def test_stack_create_total_resources_equals_max(self):
+    @mock.patch.object(stack_object.Stack, 'count_total_resources')
+    def test_stack_create_total_resources_equals_max(self, ctr):
         stack_name = 'stack_create_total_resources_equals_max'
         params = {}
         res._register_class('FakeResourceType', generic_rsrc.GenericResource)
@@ -221,6 +223,7 @@ class StackCreateTest(common.HeatTestCase):
 
         template = templatem.Template(tpl)
         stk = stack.Stack(self.ctx, stack_name, template)
+        ctr.return_value = 3
 
         mock_tmpl = self.patchobject(templatem, 'Template', return_value=stk.t)
         mock_env = self.patchobject(environment, 'Environment',
@@ -242,7 +245,8 @@ class StackCreateTest(common.HeatTestCase):
                                            parent_resource=None)
 
         self.assertEqual(stk.identifier(), result)
-        self.assertEqual(3, stk.total_resources())
+        root_stack_id = stk.root_stack_id()
+        self.assertEqual(3, stk.total_resources(root_stack_id))
         self.man.thread_group_mgr.groups[stk.id].wait()
         stk.delete()
 

heat/tests/test_engine_service.py

@@ -1063,7 +1063,8 @@ class StackServiceAdoptUpdateDeleteTest(common.HeatTestCase):
                       "('UPDATE', 'COMPLETE')",
                       six.text_type(ex.exc_info[1]))
 
-    def test_stack_update_equals(self):
+    @mock.patch.object(stack_object.Stack, 'count_total_resources')
+    def test_stack_update_equals(self, ctr):
         stack_name = 'test_stack_update_equals_resource_limit'
         params = {}
         res._register_class('GenericResourceType',
@@ -1080,6 +1081,7 @@ class StackServiceAdoptUpdateDeleteTest(common.HeatTestCase):
         sid = old_stack.store()
         old_stack.set_stack_user_project_id('1234')
         s = stack_object.Stack.get_by_id(self.ctx, sid)
+        ctr.return_value = 3
 
         stack = parser.Stack(self.ctx, stack_name, template)
 
@@ -1115,7 +1117,8 @@ class StackServiceAdoptUpdateDeleteTest(common.HeatTestCase):
         self.assertEqual(old_stack.identifier(), result)
         self.assertIsInstance(result, dict)
         self.assertTrue(result['stack_id'])
-        self.assertEqual(3, old_stack.root_stack.total_resources())
+        root_stack_id = old_stack.root_stack_id()
+        self.assertEqual(3, old_stack.total_resources(root_stack_id))
         self.m.VerifyAll()
 
     def test_stack_update_stack_id_equal(self):

heat/tests/test_nested_stack.py

@@ -84,7 +84,9 @@ Outputs:
         stack.store()
         return stack
 
-    def test_nested_stack_three_deep(self):
+    @mock.patch.object(parser.Stack, 'root_stack_id')
+    @mock.patch.object(parser.Stack, 'total_resources')
+    def test_nested_stack_three_deep(self, tr, rsi):
         root_template = '''
 HeatTemplateFormatVersion: 2012-12-12
 Resources:
@@ -116,13 +118,19 @@ Resources:
             depth2_template,
             self.nested_template]
 
+        rsi.return_value = '1234'
+        tr.return_value = 2
+
         self.validate_stack(root_template)
         calls = [mock.call('https://server.test/depth1.template'),
                  mock.call('https://server.test/depth2.template'),
                  mock.call('https://server.test/depth3.template')]
         urlfetch.get.assert_has_calls(calls)
+        tr.assert_called_with('1234')
 
-    def test_nested_stack_six_deep(self):
+    @mock.patch.object(parser.Stack, 'root_stack_id')
+    @mock.patch.object(parser.Stack, 'total_resources')
+    def test_nested_stack_six_deep(self, tr, rsi):
         tmpl = '''
 HeatTemplateFormatVersion: 2012-12-12
 Resources:
@@ -150,6 +158,9 @@ Resources:
             depth5_template,
             self.nested_template]
 
+        rsi.return_value = '1234'
+        tr.return_value = 5
+
         t = template_format.parse(root_template)
         stack = self.parse_stack(t)
         res = self.assertRaises(exception.StackValidationFailed,
@@ -202,7 +213,9 @@ Resources:
                  mock.call('https://server.test/depth4.template')]
         urlfetch.get.assert_has_calls(calls, any_order=True)
 
-    def test_nested_stack_infinite_recursion(self):
+    @mock.patch.object(parser.Stack, 'root_stack_id')
+    @mock.patch.object(parser.Stack, 'total_resources')
+    def test_nested_stack_infinite_recursion(self, tr, rsi):
         tmpl = '''
 HeatTemplateFormatVersion: 2012-12-12
 Resources:
@@ -214,6 +227,8 @@ Resources:
         urlfetch.get.return_value = tmpl
         t = template_format.parse(tmpl)
         stack = self.parse_stack(t)
+        rsi.return_value = '1234'
+        tr.return_value = 2
         res = self.assertRaises(exception.StackValidationFailed,
                                 stack.validate)
         self.assertIn('Recursion depth exceeds', six.text_type(res))

heat/tests/test_stack.py

@@ -172,39 +172,27 @@ class StackTest(common.HeatTestCase):
     def test_total_resources_empty(self):
         self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl,
                                  status_reason='flimflam')
+        self.stack.store()
+        self.assertEqual(0, self.stack.total_resources(self.stack.id))
         self.assertEqual(0, self.stack.total_resources())
 
-    def test_total_resources_generic(self):
+    def test_total_resources_not_found(self):
+        self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl,
+                                 status_reason='flimflam')
+
+        self.assertEqual(0, self.stack.total_resources('1234'))
+
+    @mock.patch.object(db_api, 'stack_count_total_resources')
+    def test_total_resources_generic(self, sctr):
         tpl = {'HeatTemplateFormatVersion': '2012-12-12',
                'Resources':
                {'A': {'Type': 'GenericResourceType'}}}
         self.stack = stack.Stack(self.ctx, 'test_stack',
                                  template.Template(tpl),
                                  status_reason='blarg')
-        self.assertEqual(1, self.stack.total_resources())
+        self.stack.store()
-
+        sctr.return_value = 1
-    def test_total_resources_nested_ok(self):
+        self.assertEqual(1, self.stack.total_resources(self.stack.id))
-        tpl = {'HeatTemplateFormatVersion': '2012-12-12',
-               'Resources':
-               {'A': {'Type': 'GenericResourceType'}}}
-        self.stack = stack.Stack(self.ctx, 'test_stack',
-                                 template.Template(tpl),
-                                 status_reason='blarg')
-
-        self.stack['A'].nested = mock.Mock()
-        self.stack['A'].nested.return_value.total_resources.return_value = 3
-        self.assertEqual(4, self.stack.total_resources())
-
-    def test_total_resources_nested_not_found(self):
-        tpl = {'HeatTemplateFormatVersion': '2012-12-12',
-               'Resources':
-               {'A': {'Type': 'GenericResourceType'}}}
-        self.stack = stack.Stack(self.ctx, 'test_stack',
-                                 template.Template(tpl),
-                                 status_reason='blarg')
-
-        self.stack['A'].nested = mock.Mock(
-            side_effect=exception.NotFound('gone'))
         self.assertEqual(1, self.stack.total_resources())
 
     def test_iter_resources(self):
@@ -264,42 +252,6 @@ class StackTest(common.HeatTestCase):
         # A cache supplied means we should never query the database.
         self.assertFalse(mock_drg.called)
 
-    def test_root_stack_no_parent(self):
-        tpl = {'HeatTemplateFormatVersion': '2012-12-12',
-               'Resources':
-               {'A': {'Type': 'GenericResourceType'}}}
-        self.stack = stack.Stack(self.ctx, 'test_stack',
-                                 template.Template(tpl),
-                                 status_reason='blarg')
-
-        self.assertEqual(self.stack, self.stack.root_stack)
-
-    def test_root_stack_parent_no_stack(self):
-        tpl = {'HeatTemplateFormatVersion': '2012-12-12',
-               'Resources':
-               {'A': {'Type': 'GenericResourceType'}}}
-        self.stack = stack.Stack(self.ctx, 'test_stack',
-                                 template.Template(tpl),
-                                 status_reason='blarg',
-                                 parent_resource='parent')
-
-        parent_resource = mock.Mock()
-        parent_resource.stack = None
-        self.stack._parent_stack = dict(parent=parent_resource)
-        self.assertEqual(self.stack, self.stack.root_stack)
-
-    def test_root_stack_with_parent(self):
-        tpl = {'HeatTemplateFormatVersion': '2012-12-12',
-               'Resources':
-               {'A': {'Type': 'GenericResourceType'}}}
-        stk = stack.Stack(self.ctx, 'test_stack', template.Template(tpl),
-                          status_reason='blarg', parent_resource='parent')
-
-        parent_resource = mock.Mock()
-        parent_resource.stack.root_stack = 'test value'
-        stk._parent_stack = dict(parent=parent_resource)
-        self.assertEqual('test value', stk.root_stack)
-
     def test_load_parent_resource(self):
         self.stack = stack.Stack(self.ctx, 'load_parent_resource', self.tmpl,
                                  parent_resource='parent')

heat/tests/test_stack_resource.py

@@ -434,7 +434,7 @@ class StackResourceTest(StackResourceBaseTest):
                 'Resources': [1]}
         template = stack_resource.template.Template(tmpl)
         root_resources = mock.Mock(return_value=2)
-        self.parent_resource.stack.root_stack.total_resources = root_resources
+        self.parent_resource.stack.total_resources = root_resources
 
         self.assertRaises(exception.RequestLimitExceeded,
                           self.parent_resource._validate_nested_resources,
@@ -534,24 +534,21 @@ class StackResourceTest(StackResourceBaseTest):
 
 class StackResourceLimitTest(StackResourceBaseTest):
     scenarios = [
-        ('1', dict(root=3, templ=4, nested=0, max=10, error=False)),
+        ('3_4_0', dict(root=3, templ=4, nested=0, max=10, error=False)),
-        ('2', dict(root=3, templ=8, nested=0, max=10, error=True)),
+        ('3_8_0', dict(root=3, templ=8, nested=0, max=10, error=True)),
-        ('3', dict(root=3, templ=8, nested=2, max=10, error=False)),
+        ('3_8_2', dict(root=3, templ=8, nested=2, max=10, error=True)),
-        ('4', dict(root=3, templ=12, nested=2, max=10, error=True))]
+        ('3_5_2', dict(root=3, templ=5, nested=2, max=10, error=False)),
+        ('3_6_2', dict(root=3, templ=6, nested=2, max=10, error=True)),
+        ('3_12_2', dict(root=3, templ=12, nested=2, max=10, error=True))]
 
     def setUp(self):
         super(StackResourceLimitTest, self).setUp()
         self.res = self.parent_resource
 
     def test_resource_limit(self):
-        # mock nested resources
-        nested = mock.MagicMock()
-        nested.resources = range(self.nested)
-        self.res.nested = mock.MagicMock(return_value=nested)
-
         # mock root total_resources
-        self.res.stack.root_stack.total_resources = mock.Mock(
+        total_resources = self.root + self.nested
-            return_value=self.root)
+        parser.Stack.total_resources = mock.Mock(return_value=total_resources)
 
         # setup the config max
         cfg.CONF.set_default('max_resources_per_stack', self.max)