From f06676970388061fcec9580966cfbc3b70d30b17 Mon Sep 17 00:00:00 2001 From: Andrew Plunk Date: Sun, 29 Sep 2013 15:46:10 -0500 Subject: [PATCH] Lazily load resources when loading a Stack Stack-list is one of heat's slowest operations. It uses the orm to fetch resources for every stack in the database, then spends a lot of time deserializing resource data. This patch forces resources and dependencies do be lazily loaded. bug 1214602 Change-Id: Ieec481cb8ab211054f8956090ad97a687c9a616b --- heat/engine/parser.py | 27 ++++++++++++++++------ heat/engine/update.py | 4 +--- heat/tests/test_engine_service.py | 37 ++++++++++++++++++++++++++++++- heat/tests/test_resource.py | 28 +++++++++++------------ heat/tests/test_validate.py | 12 +--------- 5 files changed, 72 insertions(+), 36 deletions(-) diff --git a/heat/engine/parser.py b/heat/engine/parser.py index 357a5f9614..a9b1ef074d 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -88,6 +88,8 @@ class Stack(object): self.timeout_mins = timeout_mins self.disable_rollback = disable_rollback self.parent_resource = parent_resource + self._resources = None + self._dependencies = None resources.initialise() @@ -102,12 +104,24 @@ class Stack(object): else: self.outputs = {} - template_resources = self.t[template.RESOURCES] - self.resources = dict((name, - resource.Resource(name, data, self)) - for (name, data) in template_resources.items()) + @property + def resources(self): + if self._resources is None: + template_resources = self.t[template.RESOURCES] + self._resources = dict((name, resource.Resource(name, data, self)) + for (name, data) in + template_resources.items()) + return self._resources - self.dependencies = self._get_dependencies(self.resources.itervalues()) + @property + def dependencies(self): + if self._dependencies is None: + self._dependencies = self._get_dependencies( + self.resources.itervalues()) + return self._dependencies + + def reset_dependencies(self): + self._dependencies = None @property def root_stack(self): @@ -460,8 +474,7 @@ class Stack(object): while not updater.step(): yield finally: - cur_deps = self._get_dependencies(self.resources.itervalues()) - self.dependencies = cur_deps + self.reset_dependencies() if action == self.UPDATE: reason = 'Stack successfully updated' diff --git a/heat/engine/update.py b/heat/engine/update.py index 7c8821f3d0..7616c9d629 100644 --- a/heat/engine/update.py +++ b/heat/engine/update.py @@ -65,9 +65,7 @@ class StackUpdate(object): try: yield update() finally: - prev_deps = self.previous_stack._get_dependencies( - self.previous_stack.resources.itervalues()) - self.previous_stack.dependencies = prev_deps + self.previous_stack.reset_dependencies() def _resource_update(self, res): if res.name in self.new_stack and self.new_stack[res.name] is res: diff --git a/heat/tests/test_engine_service.py b/heat/tests/test_engine_service.py index 539ab9b717..5ff38ede9f 100644 --- a/heat/tests/test_engine_service.py +++ b/heat/tests/test_engine_service.py @@ -30,6 +30,7 @@ import heat.rpc.api as engine_api import heat.db.api as db_api from heat.common import identifier from heat.common import template_format +from heat.engine import dependencies from heat.engine import parser from heat.engine.resource import _register_class from heat.engine import service @@ -44,7 +45,6 @@ from heat.tests.common import HeatTestCase from heat.tests import generic_resource as generic_rsrc from heat.tests import utils - wp_template = ''' { "AWSTemplateFormatVersion" : "2010-09-09", @@ -1768,3 +1768,38 @@ class StackServiceTest(HeatTestCase): sl = self.eng.show_stack(self.ctx, None) self.assertEqual(0, len(sl)) + + def test_lazy_load_resources(self): + stack_name = 'lazy_load_test' + res._register_class('GenericResourceType', + generic_rsrc.GenericResource) + + lazy_load_template = { + 'Resources': { + 'foo': {'Type': 'GenericResourceType'}, + 'bar': { + 'Type': 'ResourceWithPropsType', + 'Properties': { + 'Foo': {'Ref': 'foo'}, + } + } + } + } + templ = parser.Template(lazy_load_template) + stack = parser.Stack(self.ctx, stack_name, templ, + environment.Environment({})) + + self.assertEqual(stack._resources, None) + self.assertEqual(stack._dependencies, None) + + resources = stack.resources + self.assertEqual(type(resources), dict) + self.assertEqual(len(resources), 2) + self.assertEqual(type(resources.get('foo')), + generic_rsrc.GenericResource) + self.assertEqual(type(resources.get('bar')), + generic_rsrc.ResourceWithProps) + + stack_dependencies = stack.dependencies + self.assertEqual(type(stack_dependencies), dependencies.Dependencies) + self.assertEqual(len(stack_dependencies.graph()), 2) diff --git a/heat/tests/test_resource.py b/heat/tests/test_resource.py index 89e810bb81..588c6f5ac9 100644 --- a/heat/tests/test_resource.py +++ b/heat/tests/test_resource.py @@ -739,9 +739,9 @@ class ResourceDependenciesTest(HeatTestCase): } } }) + stack = parser.Stack(None, 'test', tmpl) ex = self.assertRaises(exception.InvalidTemplateReference, - parser.Stack, - None, 'test', tmpl) + getattr, stack, 'dependencies') self.assertIn('"baz" (in bar.Properties.Foo)', str(ex)) def test_hot_ref_fail(self): @@ -757,9 +757,9 @@ class ResourceDependenciesTest(HeatTestCase): } } }) + stack = parser.Stack(None, 'test', tmpl) ex = self.assertRaises(exception.InvalidTemplateReference, - parser.Stack, - None, 'test', tmpl) + getattr, stack, 'dependencies') self.assertIn('"baz" (in bar.Properties.Foo)', str(ex)) def test_getatt(self): @@ -909,9 +909,9 @@ class ResourceDependenciesTest(HeatTestCase): } } }) + stack = parser.Stack(None, 'test', tmpl) ex = self.assertRaises(exception.InvalidTemplateReference, - parser.Stack, - None, 'test', tmpl) + getattr, stack, 'dependencies') self.assertIn('"baz" (in bar.Properties.Foo)', str(ex)) def test_hot_getatt_fail(self): @@ -927,9 +927,9 @@ class ResourceDependenciesTest(HeatTestCase): } } }) + stack = parser.Stack(None, 'test', tmpl) ex = self.assertRaises(exception.InvalidTemplateReference, - parser.Stack, - None, 'test', tmpl) + getattr, stack, 'dependencies') self.assertIn('"baz" (in bar.Properties.Foo)', str(ex)) def test_getatt_fail_nested_deep(self): @@ -949,9 +949,9 @@ class ResourceDependenciesTest(HeatTestCase): } } }) + stack = parser.Stack(None, 'test', tmpl) ex = self.assertRaises(exception.InvalidTemplateReference, - parser.Stack, - None, 'test', tmpl) + getattr, stack, 'dependencies') self.assertIn('"baz" (in bar.Properties.Foo.Fn::Join[1][3])', str(ex)) def test_hot_getatt_fail_nested_deep(self): @@ -972,9 +972,9 @@ class ResourceDependenciesTest(HeatTestCase): } } }) + stack = parser.Stack(None, 'test', tmpl) ex = self.assertRaises(exception.InvalidTemplateReference, - parser.Stack, - None, 'test', tmpl) + getattr, stack, 'dependencies') self.assertIn('"baz" (in bar.Properties.Foo.Fn::Join[1][3])', str(ex)) def test_dependson(self): @@ -1005,9 +1005,9 @@ class ResourceDependenciesTest(HeatTestCase): } } }) + stack = parser.Stack(None, 'test', tmpl) ex = self.assertRaises(exception.InvalidTemplateReference, - parser.Stack, - None, 'test', tmpl) + getattr, stack, 'dependencies') self.assertIn('"wibble" (in foo)', str(ex)) diff --git a/heat/tests/test_validate.py b/heat/tests/test_validate.py index e681c0cb29..d7359bcff4 100644 --- a/heat/tests/test_validate.py +++ b/heat/tests/test_validate.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. + from testtools import skipIf from heat.engine import clients @@ -23,7 +24,6 @@ from heat.engine import resources from heat.engine.resources import instance as instances from heat.engine import service from heat.openstack.common.importutils import try_import -import heat.db.api as db_api from heat.engine import parser from heat.tests.common import HeatTestCase from heat.tests import utils @@ -551,11 +551,6 @@ class validateTest(HeatTestCase): t = template_format.parse(test_template_volumeattach % 'vdq') stack = parser.Stack(self.ctx, 'test_stack', parser.Template(t)) - self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack') - db_api.resource_get_by_name_and_stack(None, 'test_resource_name', - stack).AndReturn(None) - - self.m.ReplayAll() volumeattach = stack.resources['MountPoint'] self.assertTrue(volumeattach.validate() is None) @@ -563,11 +558,6 @@ class validateTest(HeatTestCase): t = template_format.parse(test_template_volumeattach % 'sda') stack = parser.Stack(self.ctx, 'test_stack', parser.Template(t)) - self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack') - db_api.resource_get_by_name_and_stack(None, 'test_resource_name', - stack).AndReturn(None) - - self.m.ReplayAll() volumeattach = stack.resources['MountPoint'] self.assertRaises(exception.StackValidationFailed, volumeattach.validate)