openstack
/
heat
Code Issues Proposed changes

Lazily load resources when loading a Stack 

Stack-list is one of heat's slowest operations.
It uses the orm to fetch resources for every stack
in the database, then spends a lot of time deserializing
resource data. This patch forces resources and dependencies
do be lazily loaded.

bug 1214602
Change-Id: Ieec481cb8ab211054f8956090ad97a687c9a616b
This commit is contained in:
Andrew Plunk 2013-09-29 15:46:10 -05:00
parent 31dac07a05
commit f066769703
5 changed files with 72 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
heat/engine/parser.py
View File

@ -88,6 +88,8 @@ class Stack(object):
self.timeout_mins = timeout_mins
self.disable_rollback = disable_rollback
self.parent_resource = parent_resource
self._resources = None
self._dependencies = None
resources.initialise()
@ -102,12 +104,24 @@ class Stack(object):
else:
self.outputs = {}
template_resources = self.t[template.RESOURCES]
self.resources = dict((name,
resource.Resource(name, data, self))
for (name, data) in template_resources.items())
@property
def resources(self):
if self._resources is None:
template_resources = self.t[template.RESOURCES]
self._resources = dict((name, resource.Resource(name, data, self))
for (name, data) in
template_resources.items())
return self._resources
self.dependencies = self._get_dependencies(self.resources.itervalues())
@property
def dependencies(self):
if self._dependencies is None:
self._dependencies = self._get_dependencies(
self.resources.itervalues())
return self._dependencies
def reset_dependencies(self):
self._dependencies = None
@property
def root_stack(self):
@ -460,8 +474,7 @@ class Stack(object):
while not updater.step():
yield
finally:
cur_deps = self._get_dependencies(self.resources.itervalues())
self.dependencies = cur_deps
self.reset_dependencies()
if action == self.UPDATE:
reason = 'Stack successfully updated'

4
heat/engine/update.py
View File

@ -65,9 +65,7 @@ class StackUpdate(object):
try:
yield update()
finally:
prev_deps = self.previous_stack._get_dependencies(
self.previous_stack.resources.itervalues())
self.previous_stack.dependencies = prev_deps
self.previous_stack.reset_dependencies()
def _resource_update(self, res):
if res.name in self.new_stack and self.new_stack[res.name] is res:

37
heat/tests/test_engine_service.py
View File

@ -30,6 +30,7 @@ import heat.rpc.api as engine_api
import heat.db.api as db_api
from heat.common import identifier
from heat.common import template_format
from heat.engine import dependencies
from heat.engine import parser
from heat.engine.resource import _register_class
from heat.engine import service
@ -44,7 +45,6 @@ from heat.tests.common import HeatTestCase
from heat.tests import generic_resource as generic_rsrc
from heat.tests import utils
wp_template = '''
{
"AWSTemplateFormatVersion" : "2010-09-09",
@ -1768,3 +1768,38 @@ class StackServiceTest(HeatTestCase):
sl = self.eng.show_stack(self.ctx, None)
self.assertEqual(0, len(sl))
def test_lazy_load_resources(self):
stack_name = 'lazy_load_test'
res._register_class('GenericResourceType',
generic_rsrc.GenericResource)
lazy_load_template = {
'Resources': {
'foo': {'Type': 'GenericResourceType'},
'bar': {
'Type': 'ResourceWithPropsType',
'Properties': {
'Foo': {'Ref': 'foo'},
}
}
}
}
templ = parser.Template(lazy_load_template)
stack = parser.Stack(self.ctx, stack_name, templ,
environment.Environment({}))
self.assertEqual(stack._resources, None)
self.assertEqual(stack._dependencies, None)
resources = stack.resources
self.assertEqual(type(resources), dict)
self.assertEqual(len(resources), 2)
self.assertEqual(type(resources.get('foo')),
generic_rsrc.GenericResource)
self.assertEqual(type(resources.get('bar')),
generic_rsrc.ResourceWithProps)
stack_dependencies = stack.dependencies
self.assertEqual(type(stack_dependencies), dependencies.Dependencies)
self.assertEqual(len(stack_dependencies.graph()), 2)

28
heat/tests/test_resource.py
View File

@ -739,9 +739,9 @@ class ResourceDependenciesTest(HeatTestCase):
}
}
})
stack = parser.Stack(None, 'test', tmpl)
ex = self.assertRaises(exception.InvalidTemplateReference,
parser.Stack,
None, 'test', tmpl)
getattr, stack, 'dependencies')
self.assertIn('"baz" (in bar.Properties.Foo)', str(ex))
def test_hot_ref_fail(self):
@ -757,9 +757,9 @@ class ResourceDependenciesTest(HeatTestCase):
}
}
})
stack = parser.Stack(None, 'test', tmpl)
ex = self.assertRaises(exception.InvalidTemplateReference,
parser.Stack,
None, 'test', tmpl)
getattr, stack, 'dependencies')
self.assertIn('"baz" (in bar.Properties.Foo)', str(ex))
def test_getatt(self):
@ -909,9 +909,9 @@ class ResourceDependenciesTest(HeatTestCase):
}
}
})
stack = parser.Stack(None, 'test', tmpl)
ex = self.assertRaises(exception.InvalidTemplateReference,
parser.Stack,
None, 'test', tmpl)
getattr, stack, 'dependencies')
self.assertIn('"baz" (in bar.Properties.Foo)', str(ex))
def test_hot_getatt_fail(self):
@ -927,9 +927,9 @@ class ResourceDependenciesTest(HeatTestCase):
}
}
})
stack = parser.Stack(None, 'test', tmpl)
ex = self.assertRaises(exception.InvalidTemplateReference,
parser.Stack,
None, 'test', tmpl)
getattr, stack, 'dependencies')
self.assertIn('"baz" (in bar.Properties.Foo)', str(ex))
def test_getatt_fail_nested_deep(self):
@ -949,9 +949,9 @@ class ResourceDependenciesTest(HeatTestCase):
}
}
})
stack = parser.Stack(None, 'test', tmpl)
ex = self.assertRaises(exception.InvalidTemplateReference,
parser.Stack,
None, 'test', tmpl)
getattr, stack, 'dependencies')
self.assertIn('"baz" (in bar.Properties.Foo.Fn::Join[1][3])', str(ex))
def test_hot_getatt_fail_nested_deep(self):
@ -972,9 +972,9 @@ class ResourceDependenciesTest(HeatTestCase):
}
}
})
stack = parser.Stack(None, 'test', tmpl)
ex = self.assertRaises(exception.InvalidTemplateReference,
parser.Stack,
None, 'test', tmpl)
getattr, stack, 'dependencies')
self.assertIn('"baz" (in bar.Properties.Foo.Fn::Join[1][3])', str(ex))
def test_dependson(self):
@ -1005,9 +1005,9 @@ class ResourceDependenciesTest(HeatTestCase):
}
}
})
stack = parser.Stack(None, 'test', tmpl)
ex = self.assertRaises(exception.InvalidTemplateReference,
parser.Stack,
None, 'test', tmpl)
getattr, stack, 'dependencies')
self.assertIn('"wibble" (in foo)', str(ex))

12
heat/tests/test_validate.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from testtools import skipIf
from heat.engine import clients
@ -23,7 +24,6 @@ from heat.engine import resources
from heat.engine.resources import instance as instances
from heat.engine import service
from heat.openstack.common.importutils import try_import
import heat.db.api as db_api
from heat.engine import parser
from heat.tests.common import HeatTestCase
from heat.tests import utils
@ -551,11 +551,6 @@ class validateTest(HeatTestCase):
t = template_format.parse(test_template_volumeattach % 'vdq')
stack = parser.Stack(self.ctx, 'test_stack', parser.Template(t))
self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack')
db_api.resource_get_by_name_and_stack(None, 'test_resource_name',
stack).AndReturn(None)
self.m.ReplayAll()
volumeattach = stack.resources['MountPoint']
self.assertTrue(volumeattach.validate() is None)
@ -563,11 +558,6 @@ class validateTest(HeatTestCase):
t = template_format.parse(test_template_volumeattach % 'sda')
stack = parser.Stack(self.ctx, 'test_stack', parser.Template(t))
self.m.StubOutWithMock(db_api, 'resource_get_by_name_and_stack')
db_api.resource_get_by_name_and_stack(None, 'test_resource_name',
stack).AndReturn(None)
self.m.ReplayAll()
volumeattach = stack.resources['MountPoint']
self.assertRaises(exception.StackValidationFailed,
volumeattach.validate)