Sleep between check_resource calls

Instead of calling check_resource on all leaves in the resource graph at
once, sleep a little bit between each call. As it's a tad slower,
delegate it to a thread so that the stack_create RPC message doesn't
timeout when you have lots of resources.

Change-Id: I84d2b34d65b3ce7d8d858de106dac531aff509b7
Partial-Bug: #1566845
This commit is contained in:
Thomas Herve 2016-11-21 10:43:56 +01:00
parent 990f484962
commit 4a500125b3
8 changed files with 25 additions and 3 deletions

View File

@ -1268,7 +1268,7 @@ class Stack(collections.Mapping):
# TODO(later): lifecycle_plugin_utils.do_pre_ops
self._converge_create_or_update()
self.thread_group_mgr.start(self.id, self._converge_create_or_update)
def _converge_create_or_update(self):
current_resources = self._update_or_store_resources()
@ -1322,6 +1322,8 @@ class Stack(collections.Mapping):
self.current_traversal,
input_data, is_update,
self.adopt_stack_data)
if scheduler.ENABLE_SLEEP:
eventlet.sleep(1)
def rollback(self):
old_tmpl_id = self.prev_raw_template_id

View File

@ -114,4 +114,5 @@ class Engine(message_processor.MessageProcessor):
cntxt = utils.dummy_context()
db_stack = db_api.stack_get_by_name(cntxt, stack_name)
stk = stack.Stack.load(cntxt, stack=db_stack)
stk.thread_group_mgr = SynchronousThreadGroupManager()
stk.rollback()

View File

@ -547,6 +547,7 @@ class CheckWorkflowCleanupTest(common.HeatTestCase):
template=tools.string_template_five, convergence=True)
tstack.converge_stack(tstack.t, action=tstack.CREATE)
self.stack = stack.Stack.load(self.ctx, stack_id=tstack.id)
self.stack.thread_group_mgr = tools.DummyThreadGroupManager()
self.stack.converge_stack(self.stack.t, action=self.stack.DELETE)
self.resource = self.stack['A']
self.is_update = False

View File

@ -164,6 +164,7 @@ def get_stack(stack_name, ctx, template=None, with_params=True,
tmpl = templatem.Template(t)
stack = parser.Stack(ctx, stack_name, tmpl, convergence=convergence,
**kwargs)
stack.thread_group_mgr = DummyThreadGroupManager()
return stack
@ -323,6 +324,11 @@ class DummyThreadGroupManager(object):
self.msg_queues = []
self.messages = []
def start(self, stack, func, *args, **kwargs):
# Just run the function, so we know it's completed in the test
func(*args, **kwargs)
return DummyThread()
def start_with_lock(self, cnxt, stack, engine_id, func, *args, **kwargs):
# Just run the function, so we know it's completed in the test
func(*args, **kwargs)

View File

@ -43,6 +43,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
stack = parser.Stack(utils.dummy_context(), 'empty_tmpl_stack',
empty_tmpl, convergence=True)
stack.store()
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(template=stack.t, action=stack.CREATE)
self.assertFalse(mock_cr.called)
mock_mc.assert_called_once_with()
@ -175,6 +176,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
with mock.patch.object(
parser.Stack, 'db_active_resources_get',
side_effect=self._mock_convg_db_update_requires):
curr_stack.thread_group_mgr = tools.DummyThreadGroupManager()
curr_stack.converge_stack(template=template2, action=stack.UPDATE)
self.assertIsNotNone(curr_stack.ext_rsrcs_db)
@ -300,6 +302,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
with mock.patch.object(
parser.Stack, 'db_active_resources_get',
side_effect=self._mock_convg_db_update_requires):
curr_stack.thread_group_mgr = tools.DummyThreadGroupManager()
curr_stack.converge_stack(template=template2, action=stack.DELETE)
self.assertIsNotNone(curr_stack.ext_rsrcs_db)
@ -507,6 +510,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
stack = parser.Stack(utils.dummy_context(), 'convg_updated_time_test',
templatem.Template.create_empty_template(),
convergence=True)
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(template=stack.t, action=stack.CREATE)
self.assertIsNone(stack.updated_time)
self.assertTrue(mock_ccu.called)
@ -517,6 +521,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
'Resources': {'R1': {'Type': 'GenericResourceType'}}}
stack = parser.Stack(utils.dummy_context(), 'updated_time_test',
templatem.Template(tmpl), convergence=True)
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(template=stack.t, action=stack.UPDATE)
self.assertIsNotNone(stack.updated_time)
self.assertTrue(mock_ccu.called)
@ -528,6 +533,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
mock_ccu, mock_cr):
stack = parser.Stack(utils.dummy_context(), 'convg_updated_time_test',
templatem.Template.create_empty_template())
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(template=stack.t, action=stack.CREATE)
self.assertFalse(mock_syncpoint_del.called)
self.assertTrue(mock_ccu.called)
@ -541,6 +547,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
'Resources': {'R1': {'Type': 'GenericResourceType'}}}
stack = parser.Stack(utils.dummy_context(), 'updated_time_test',
templatem.Template(tmpl))
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.current_traversal = 'prev_traversal'
stack.converge_stack(template=stack.t, action=stack.UPDATE)
self.assertTrue(mock_syncpoint_del.called)
@ -554,6 +561,7 @@ class StackConvergenceCreateUpdateDeleteTest(common.HeatTestCase):
stack.current_traversal = 'prev_traversal'
stack.action, stack.status = stack.CREATE, stack.COMPLETE
stack.store()
stack.thread_group_mgr = tools.DummyThreadGroupManager()
snapshot_values = {
'stack_id': stack.id,
'name': 'fake_snapshot',

View File

@ -211,6 +211,7 @@ class StackConvergenceServiceCreateUpdateTest(common.HeatTestCase):
cfg.CONF.set_override('convergence_engine', True, enforce_type=True)
self.ctx = utils.dummy_context()
self.man = service.EngineService('a-host', 'a-topic')
self.man.thread_group_mgr = tools.DummyThreadGroupManager()
def _stub_update_mocks(self, stack_to_load, stack_to_return):
self.m.StubOutWithMock(parser, 'Stack')

View File

@ -52,6 +52,7 @@ from heat.objects import resource as resource_objects
from heat.objects import resource_data as resource_data_object
from heat.objects import resource_properties_data as rpd_object
from heat.tests import common
from heat.tests.engine import tools
from heat.tests import generic_resource as generic_rsrc
from heat.tests import utils
@ -2079,6 +2080,7 @@ class ResourceTest(common.HeatTestCase):
}}, env=self.env)
stack = parser.Stack(utils.dummy_context(), 'test_stack',
tmpl)
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(stack.t, action=stack.CREATE)
res = stack.resources['test_res']
res.requires = [2]
@ -2117,6 +2119,7 @@ class ResourceTest(common.HeatTestCase):
}}, env=self.env)
stack = parser.Stack(utils.dummy_context(), 'test_stack',
tmpl)
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(stack.t, action=stack.CREATE)
res = stack.resources['test_res']
res.store()
@ -2211,6 +2214,7 @@ class ResourceTest(common.HeatTestCase):
}}, env=self.env)
stack = parser.Stack(utils.dummy_context(), 'test_stack',
tmpl)
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(stack.t, action=stack.CREATE)
res = stack.resources['test_res']
res.requires = [2]
@ -2254,6 +2258,7 @@ class ResourceTest(common.HeatTestCase):
}}, env=self.env)
stack = parser.Stack(utils.dummy_context(), 'test_stack',
tmpl)
stack.thread_group_mgr = tools.DummyThreadGroupManager()
stack.converge_stack(stack.t, action=stack.CREATE)
res = stack.resources['test_res']
res.requires = [2]

View File

@ -29,8 +29,6 @@ echo -e 'notification_driver=messagingv2\n' >> $localconf
echo -e 'hidden_stack_tags=hidden\n' >> $localconf
echo -e 'encrypt_parameters_and_properties=True\n' >> $localconf
echo -e 'logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s\n' >> $localconf
# Limit the number of connections, we're overflowing mysql
echo -e 'executor_thread_pool_size=8\n' >> $localconf
echo -e '[heat_api]\nworkers=2\n' >> $localconf
echo -e '[heat_api_cfn]\nworkers=2\n' >> $localconf