Convergence: Implementation of timeout
The resource provisioning work is distributed among heat engines, so the timeout also has to be distributed and brought to the resource level granularity. Thus, 1. Before invoking check_resource on a resource, ensure that the stack has not timed out. 2. Pass the remaining amount of time to the resource converge method so that it can raise timeout exception if it cannot finish in the remaining time. Once timeout exception is raised by a resource converge method, the corresponding stack is marked as FAILED with "Timed out" as failure reason. Then, if rollback is enabled on the stack, it is triggered. Change-Id: Id1806d546c67505137f57f72d5b463dc229a666d
This commit is contained in:
Anant Patil
parent
850963839c
commit
b5968ef068
|
|
@ -697,7 +697,8 @@ class Resource(object):
|
|||
'''
|
||||
return self
|
||||
|
||||
def create_convergence(self, template_id, resource_data, engine_id):
|
||||
def create_convergence(self, template_id, resource_data, engine_id,
|
||||
timeout):
|
||||
'''
|
||||
Creates the resource by invoking the scheduler TaskRunner.
|
||||
'''
|
||||
|
|
@ -712,7 +713,7 @@ class Resource(object):
|
|||
else:
|
||||
adopt_data = self.stack._adopt_kwargs(self)
|
||||
runner = scheduler.TaskRunner(self.adopt, **adopt_data)
|
||||
runner()
|
||||
runner(timeout=timeout)
|
||||
|
||||
@scheduler.wrappertask
|
||||
def create(self):
|
||||
|
|
@ -865,7 +866,8 @@ class Resource(object):
|
|||
except ValueError:
|
||||
return True
|
||||
|
||||
def update_convergence(self, template_id, resource_data, engine_id):
|
||||
def update_convergence(self, template_id, resource_data, engine_id,
|
||||
timeout):
|
||||
'''
|
||||
Updates the resource by invoking the scheduler TaskRunner
|
||||
and it persists the resource's current_template_id to template_id and
|
||||
|
|
@ -876,7 +878,7 @@ class Resource(object):
|
|||
new_temp = template.Template.load(self.context, template_id)
|
||||
new_res_def = new_temp.resource_definitions(self.stack)[self.name]
|
||||
runner = scheduler.TaskRunner(self.update, new_res_def)
|
||||
runner()
|
||||
runner(timeout=timeout)
|
||||
|
||||
# update the resource db record (stored in unlock)
|
||||
self.current_template_id = template_id
|
||||
|
|
@ -1107,7 +1109,7 @@ class Resource(object):
|
|||
expected_engine_id=None
|
||||
)
|
||||
|
||||
def delete_convergence(self, template_id, input_data, engine_id):
|
||||
def delete_convergence(self, template_id, input_data, engine_id, timeout):
|
||||
'''Destroys the resource if it doesn't belong to given
|
||||
template. The given template is suppose to be the current
|
||||
template being provisioned.
|
||||
|
|
@ -1124,7 +1126,7 @@ class Resource(object):
|
|||
|
||||
if self.current_template_id != template_id:
|
||||
runner = scheduler.TaskRunner(self.destroy)
|
||||
runner()
|
||||
runner(timeout=timeout)
|
||||
|
||||
# update needed_by and replaces of replacement resource
|
||||
self._update_replacement_data(template_id)
|
||||
|
|
|
|||
|
|
@ -1655,3 +1655,25 @@ class Stack(collections.Mapping):
|
|||
stack_object.Stack.delete(self.context, self.id)
|
||||
except exception.NotFound:
|
||||
pass
|
||||
|
||||
def time_elapsed(self):
|
||||
'''
|
||||
Time elapsed in seconds since the stack operation started.
|
||||
'''
|
||||
start_time = self.updated_time or self.created_time
|
||||
return (datetime.datetime.utcnow() - start_time).seconds
|
||||
|
||||
def time_remaining(self):
|
||||
'''
|
||||
Time left before stack times out.
|
||||
'''
|
||||
return self.timeout_secs() - self.time_elapsed()
|
||||
|
||||
def has_timed_out(self):
|
||||
'''
|
||||
Returns True if this stack has timed-out.
|
||||
'''
|
||||
if self.status == self.IN_PROGRESS:
|
||||
return self.time_elapsed() > self.timeout_secs()
|
||||
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from heat.common.i18n import _LE
|
|||
from heat.common.i18n import _LI
|
||||
from heat.common import messaging as rpc_messaging
|
||||
from heat.engine import resource
|
||||
from heat.engine import scheduler
|
||||
from heat.engine import stack as parser
|
||||
from heat.engine import sync_point
|
||||
from heat.objects import resource as resource_objects
|
||||
|
|
@ -107,13 +108,7 @@ class WorkerService(service.Service):
|
|||
{'action': stack.action, 'stack_name': stack.name})
|
||||
stack.rollback()
|
||||
|
||||
def _handle_resource_failure(self, cnxt, stack_id, traversal_id,
|
||||
failure_reason):
|
||||
stack = parser.Stack.load(cnxt, stack_id=stack_id)
|
||||
# make sure no new stack operation was triggered
|
||||
if stack.current_traversal != traversal_id:
|
||||
return
|
||||
|
||||
def _handle_failure(self, cnxt, stack, failure_reason):
|
||||
stack.state_set(stack.action, stack.FAILED, failure_reason)
|
||||
|
||||
if (not stack.disable_rollback and
|
||||
|
|
@ -122,6 +117,19 @@ class WorkerService(service.Service):
|
|||
else:
|
||||
stack.purge_db()
|
||||
|
||||
def _handle_resource_failure(self, cnxt, stack_id, traversal_id,
|
||||
failure_reason):
|
||||
stack = parser.Stack.load(cnxt, stack_id=stack_id)
|
||||
# make sure no new stack operation was triggered
|
||||
if stack.current_traversal != traversal_id:
|
||||
return
|
||||
|
||||
self._handle_failure(cnxt, stack, failure_reason)
|
||||
|
||||
def _handle_stack_timeout(self, cnxt, stack):
|
||||
failure_reason = u'Timed out'
|
||||
self._handle_failure(cnxt, stack, failure_reason)
|
||||
|
||||
def _load_resource(self, cnxt, resource_id, resource_data, is_update):
|
||||
if is_update:
|
||||
cache_data = {in_data.get(
|
||||
|
|
@ -141,12 +149,13 @@ class WorkerService(service.Service):
|
|||
return rsrc, stack
|
||||
|
||||
def _do_check_resource(self, cnxt, current_traversal, tmpl, resource_data,
|
||||
is_update, rsrc, stack_id, adopt_stack_data):
|
||||
is_update, rsrc, stack, adopt_stack_data):
|
||||
try:
|
||||
if is_update:
|
||||
try:
|
||||
check_resource_update(rsrc, tmpl.id, resource_data,
|
||||
self.engine_id)
|
||||
self.engine_id,
|
||||
stack.time_remaining())
|
||||
except resource.UpdateReplace:
|
||||
new_res_id = rsrc.make_replacement(tmpl.id)
|
||||
LOG.info("Replacing resource with new id %s", new_res_id)
|
||||
|
|
@ -160,7 +169,7 @@ class WorkerService(service.Service):
|
|||
|
||||
else:
|
||||
check_resource_cleanup(rsrc, tmpl.id, resource_data,
|
||||
self.engine_id)
|
||||
self.engine_id, stack.time_remaining())
|
||||
|
||||
return True
|
||||
except resource.UpdateInProgress:
|
||||
|
|
@ -175,7 +184,13 @@ class WorkerService(service.Service):
|
|||
reason = 'Resource %s failed: %s' % (rsrc.action,
|
||||
six.text_type(ex))
|
||||
self._handle_resource_failure(
|
||||
cnxt, stack_id, current_traversal, reason)
|
||||
cnxt, stack.id, current_traversal, reason)
|
||||
except scheduler.Timeout:
|
||||
# reload the stack to verify current traversal
|
||||
stack = parser.Stack.load(cnxt, stack_id=stack.id)
|
||||
if stack.current_traversal != current_traversal:
|
||||
return
|
||||
self._handle_stack_timeout(cnxt, stack)
|
||||
|
||||
return False
|
||||
|
||||
|
|
@ -267,6 +282,10 @@ class WorkerService(service.Service):
|
|||
LOG.debug('[%s] Traversal cancelled; stopping.', current_traversal)
|
||||
return
|
||||
|
||||
if stack.has_timed_out():
|
||||
self._handle_stack_timeout(cnxt, stack)
|
||||
return
|
||||
|
||||
tmpl = stack.t
|
||||
stack.adopt_stack_data = adopt_stack_data
|
||||
|
||||
|
|
@ -278,7 +297,7 @@ class WorkerService(service.Service):
|
|||
check_resource_done = self._do_check_resource(cnxt, current_traversal,
|
||||
tmpl, resource_data,
|
||||
is_update,
|
||||
rsrc, stack.id,
|
||||
rsrc, stack,
|
||||
adopt_stack_data)
|
||||
|
||||
if check_resource_done:
|
||||
|
|
@ -343,18 +362,20 @@ def propagate_check_resource(cnxt, rpc_client, next_res_id,
|
|||
{sender_key: sender_data})
|
||||
|
||||
|
||||
def check_resource_update(rsrc, template_id, resource_data, engine_id):
|
||||
def check_resource_update(rsrc, template_id, resource_data, engine_id,
|
||||
timeout):
|
||||
'''
|
||||
Create or update the Resource if appropriate.
|
||||
'''
|
||||
if rsrc.action == resource.Resource.INIT:
|
||||
rsrc.create_convergence(template_id, resource_data, engine_id)
|
||||
rsrc.create_convergence(template_id, resource_data, engine_id, timeout)
|
||||
else:
|
||||
rsrc.update_convergence(template_id, resource_data, engine_id)
|
||||
rsrc.update_convergence(template_id, resource_data, engine_id, timeout)
|
||||
|
||||
|
||||
def check_resource_cleanup(rsrc, template_id, resource_data, engine_id):
|
||||
def check_resource_cleanup(rsrc, template_id, resource_data, engine_id,
|
||||
timeout):
|
||||
'''
|
||||
Delete the Resource if appropriate.
|
||||
'''
|
||||
rsrc.delete_convergence(template_id, resource_data, engine_id)
|
||||
rsrc.delete_convergence(template_id, resource_data, engine_id, timeout)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import mock
|
|||
|
||||
from heat.common import exception
|
||||
from heat.engine import resource
|
||||
from heat.engine import scheduler
|
||||
from heat.engine import stack
|
||||
from heat.engine import sync_point
|
||||
from heat.engine import worker
|
||||
|
|
@ -146,7 +147,8 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
|||
self.is_update, None)
|
||||
mock_cru.assert_called_once_with(self.resource,
|
||||
self.resource.stack.t.id,
|
||||
{}, self.worker.engine_id)
|
||||
{}, self.worker.engine_id,
|
||||
mock.ANY)
|
||||
self.assertFalse(mock_crc.called)
|
||||
|
||||
expected_calls = []
|
||||
|
|
@ -171,7 +173,8 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
|||
self.is_update, None)
|
||||
mock_cru.assert_called_once_with(self.resource,
|
||||
self.resource.stack.t.id,
|
||||
{}, self.worker.engine_id)
|
||||
{}, self.worker.engine_id,
|
||||
self.stack.timeout_secs())
|
||||
self.assertTrue(mock_mr.called)
|
||||
self.assertFalse(mock_crc.called)
|
||||
self.assertFalse(mock_pcr.called)
|
||||
|
|
@ -188,7 +191,8 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
|||
self.is_update, None)
|
||||
mock_cru.assert_called_once_with(self.resource,
|
||||
self.resource.stack.t.id,
|
||||
{}, self.worker.engine_id)
|
||||
{}, self.worker.engine_id,
|
||||
self.stack.timeout_secs())
|
||||
self.assertFalse(mock_crc.called)
|
||||
self.assertFalse(mock_pcr.called)
|
||||
self.assertFalse(mock_csc.called)
|
||||
|
|
@ -400,6 +404,62 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
|||
actual_predecessors = call_args[5]
|
||||
self.assertItemsEqual(expected_predecessors, actual_predecessors)
|
||||
|
||||
@mock.patch.object(stack.Stack, 'purge_db')
|
||||
def test_handle_failure(self, mock_purgedb, mock_cru, mock_crc, mock_pcr,
|
||||
mock_csc, mock_cid):
|
||||
self.worker._handle_failure(self.ctx, self.stack, 'dummy-reason')
|
||||
mock_purgedb.assert_called_once_with()
|
||||
self.assertEqual('dummy-reason', self.stack.status_reason)
|
||||
|
||||
# test with rollback
|
||||
self.worker._trigger_rollback = mock.Mock()
|
||||
self.stack.disable_rollback = False
|
||||
self.stack.state_set(self.stack.UPDATE, self.stack.IN_PROGRESS, '')
|
||||
self.worker._handle_failure(self.ctx, self.stack, 'dummy-reason')
|
||||
self.worker._trigger_rollback.assert_called_once_with(self.stack)
|
||||
|
||||
def test_handle_stack_timeout(self, mock_cru, mock_crc, mock_pcr,
|
||||
mock_csc, mock_cid):
|
||||
self.worker._handle_failure = mock.Mock()
|
||||
self.worker._handle_stack_timeout(self.ctx, self.stack)
|
||||
self.worker._handle_failure.assert_called_once_with(
|
||||
self.ctx, self.stack, u'Timed out')
|
||||
|
||||
def test_do_check_resource_marks_stack_as_failed_if_stack_timesout(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
mock_cru.side_effect = scheduler.Timeout(None, 60)
|
||||
self.is_update = True
|
||||
self.worker._handle_stack_timeout = mock.Mock()
|
||||
self.worker._do_check_resource(self.ctx, self.stack.current_traversal,
|
||||
self.stack.t, {}, self.is_update,
|
||||
self.resource, self.stack, {})
|
||||
self.worker._handle_stack_timeout.assert_called_once_with(
|
||||
self.ctx, self.stack)
|
||||
|
||||
def test_do_check_resource_ignores_timeout_for_new_update(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
# Ensure current_traversal is check before marking the stack as
|
||||
# failed due to time-out.
|
||||
mock_cru.side_effect = scheduler.Timeout(None, 60)
|
||||
self.is_update = True
|
||||
self.worker._handle_stack_timeout = mock.Mock()
|
||||
old_traversal = self.stack.current_traversal
|
||||
self.stack.current_traversal = 'new_traversal'
|
||||
self.worker._do_check_resource(self.ctx, old_traversal,
|
||||
self.stack.t, {}, self.is_update,
|
||||
self.resource, self.stack, {})
|
||||
self.assertFalse(self.worker._handle_stack_timeout.called)
|
||||
|
||||
@mock.patch.object(stack.Stack, 'has_timed_out')
|
||||
def test_check_resource_handles_timeout(self, mock_to, mock_cru, mock_crc,
|
||||
mock_pcr, mock_csc, mock_cid):
|
||||
mock_to.return_value = True
|
||||
self.worker._handle_stack_timeout = mock.Mock()
|
||||
self.worker.check_resource(self.ctx, self.resource.id,
|
||||
self.stack.current_traversal,
|
||||
{}, self.is_update, {})
|
||||
self.assertTrue(self.worker._handle_stack_timeout.called)
|
||||
|
||||
|
||||
@mock.patch.object(worker, 'construct_input_data')
|
||||
@mock.patch.object(worker, 'check_stack_complete')
|
||||
|
|
@ -436,7 +496,8 @@ class CheckWorkflowCleanupTest(common.HeatTestCase):
|
|||
self.assertFalse(mock_cru.called)
|
||||
mock_crc.assert_called_once_with(
|
||||
self.resource, self.resource.stack.t.id,
|
||||
{}, self.worker.engine_id)
|
||||
{}, self.worker.engine_id,
|
||||
self.stack.timeout_secs())
|
||||
|
||||
def test_is_cleanup_traversal_raise_update_inprogress(
|
||||
self, mock_cru, mock_crc, mock_pcr, mock_csc, mock_cid):
|
||||
|
|
@ -446,7 +507,8 @@ class CheckWorkflowCleanupTest(common.HeatTestCase):
|
|||
self.is_update, None)
|
||||
mock_crc.assert_called_once_with(self.resource,
|
||||
self.resource.stack.t.id,
|
||||
{}, self.worker.engine_id)
|
||||
{}, self.worker.engine_id,
|
||||
self.stack.timeout_secs())
|
||||
self.assertFalse(mock_cru.called)
|
||||
self.assertFalse(mock_pcr.called)
|
||||
self.assertFalse(mock_csc.called)
|
||||
|
|
@ -512,7 +574,8 @@ class MiscMethodsTest(common.HeatTestCase):
|
|||
def test_check_resource_update_init_action(self, mock_update, mock_create):
|
||||
self.resource.action = 'INIT'
|
||||
worker.check_resource_update(self.resource, self.resource.stack.t.id,
|
||||
{}, 'engine-id')
|
||||
{}, 'engine-id',
|
||||
self.stack.timeout_secs())
|
||||
self.assertTrue(mock_create.called)
|
||||
self.assertFalse(mock_update.called)
|
||||
|
||||
|
|
@ -522,7 +585,8 @@ class MiscMethodsTest(common.HeatTestCase):
|
|||
self, mock_update, mock_create):
|
||||
self.resource.action = 'CREATE'
|
||||
worker.check_resource_update(self.resource, self.resource.stack.t.id,
|
||||
{}, 'engine-id')
|
||||
{}, 'engine-id',
|
||||
self.stack.timeout_secs())
|
||||
self.assertFalse(mock_create.called)
|
||||
self.assertTrue(mock_update.called)
|
||||
|
||||
|
|
@ -532,7 +596,8 @@ class MiscMethodsTest(common.HeatTestCase):
|
|||
self, mock_update, mock_create):
|
||||
self.resource.action = 'UPDATE'
|
||||
worker.check_resource_update(self.resource, self.resource.stack.t.id,
|
||||
{}, 'engine-id')
|
||||
{}, 'engine-id',
|
||||
self.stack.timeout_secs())
|
||||
self.assertFalse(mock_create.called)
|
||||
self.assertTrue(mock_update.called)
|
||||
|
||||
|
|
@ -540,5 +605,6 @@ class MiscMethodsTest(common.HeatTestCase):
|
|||
def test_check_resource_cleanup_delete(self, mock_delete):
|
||||
self.resource.current_template_id = 'new-template-id'
|
||||
worker.check_resource_cleanup(self.resource, self.resource.stack.t.id,
|
||||
{}, 'engine-id')
|
||||
{}, 'engine-id',
|
||||
self.stack.timeout_secs())
|
||||
self.assertTrue(mock_delete.called)
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ class ResourceTest(common.HeatTestCase):
|
|||
template.Template(empty_template,
|
||||
env=self.env),
|
||||
stack_id=str(uuid.uuid4()))
|
||||
self.dummy_timeout = 10
|
||||
|
||||
def test_get_class_ok(self):
|
||||
cls = resources.global_env().get_class('GenericResourceType')
|
||||
|
|
@ -1466,8 +1467,10 @@ class ResourceTest(common.HeatTestCase):
|
|||
self.assertEqual(engine_id, rs.engine_id)
|
||||
self.assertEqual(atomic_key, rs.atomic_key)
|
||||
|
||||
@mock.patch.object(resource.Resource, 'create')
|
||||
def test_create_convergence(self, mock_create):
|
||||
@mock.patch.object(resource.scheduler.TaskRunner, '__init__',
|
||||
return_value=None)
|
||||
@mock.patch.object(resource.scheduler.TaskRunner, '__call__')
|
||||
def test_create_convergence(self, mock_call, mock_init):
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
|
||||
res = generic_rsrc.GenericResource('test_res', tmpl, self.stack)
|
||||
res.action = res.CREATE
|
||||
|
|
@ -1475,13 +1478,28 @@ class ResourceTest(common.HeatTestCase):
|
|||
self._assert_resource_lock(res.id, None, None)
|
||||
res_data = {(1, True): {u'id': 1, u'name': 'A', 'attrs': {}},
|
||||
(2, True): {u'id': 3, u'name': 'B', 'attrs': {}}}
|
||||
res.create_convergence(self.stack.t.id, res_data, 'engine-007')
|
||||
|
||||
mock_create.assert_called_once_with()
|
||||
res.create_convergence(self.stack.t.id, res_data, 'engine-007',
|
||||
60)
|
||||
|
||||
mock_init.assert_called_once_with(res.create)
|
||||
mock_call.assert_called_once_with(timeout=60)
|
||||
self.assertEqual(self.stack.t.id, res.current_template_id)
|
||||
self.assertItemsEqual([1, 3], res.requires)
|
||||
self._assert_resource_lock(res.id, None, 2)
|
||||
|
||||
def test_create_convergence_throws_timeout(self):
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
|
||||
res = generic_rsrc.GenericResource('test_res', tmpl, self.stack)
|
||||
res.action = res.CREATE
|
||||
res._store()
|
||||
res_data = {(1, True): {u'id': 1, u'name': 'A', 'attrs': {}},
|
||||
(2, True): {u'id': 3, u'name': 'B', 'attrs': {}}}
|
||||
|
||||
self.assertRaises(scheduler.Timeout, res.create_convergence,
|
||||
self.stack.t.id, res_data, 'engine-007',
|
||||
0)
|
||||
|
||||
def test_create_convergence_sets_requires_for_failure(self):
|
||||
'''
|
||||
Ensure that requires are computed correctly even if resource
|
||||
|
|
@ -1497,7 +1515,7 @@ class ResourceTest(common.HeatTestCase):
|
|||
(2, True): {u'id': 3, u'name': 'B', 'attrs': {}}}
|
||||
self.assertRaises(exception.ResourceNotAvailable,
|
||||
res.create_convergence, self.stack.t.id, res_data,
|
||||
'engine-007')
|
||||
'engine-007', self.dummy_timeout)
|
||||
self.assertItemsEqual([5, 3], res.requires)
|
||||
self._assert_resource_lock(res.id, None, 2)
|
||||
|
||||
|
|
@ -1512,7 +1530,8 @@ class ResourceTest(common.HeatTestCase):
|
|||
self._assert_resource_lock(res.id, None, None)
|
||||
res_data = {(1, True): {u'id': 5, u'name': 'A', 'attrs': {}},
|
||||
(2, True): {u'id': 3, u'name': 'B', 'attrs': {}}}
|
||||
res.create_convergence(self.stack.t.id, res_data, 'engine-007')
|
||||
res.create_convergence(self.stack.t.id, res_data, 'engine-007',
|
||||
self.dummy_timeout)
|
||||
|
||||
mock_adopt.assert_called_once_with(
|
||||
resource_data={'resource_id': 'fluffy'})
|
||||
|
|
@ -1530,11 +1549,13 @@ class ResourceTest(common.HeatTestCase):
|
|||
(2, True): {u'id': 3, u'name': 'B', 'attrs': {}}}
|
||||
exc = self.assertRaises(exception.ResourceFailure,
|
||||
res.create_convergence, self.stack.t.id,
|
||||
res_data, 'engine-007')
|
||||
res_data, 'engine-007', self.dummy_timeout)
|
||||
self.assertIn('Resource ID was not provided', six.text_type(exc))
|
||||
|
||||
@mock.patch.object(resource.Resource, 'update')
|
||||
def test_update_convergence(self, mock_update):
|
||||
@mock.patch.object(resource.scheduler.TaskRunner, '__init__',
|
||||
return_value=None)
|
||||
@mock.patch.object(resource.scheduler.TaskRunner, '__call__')
|
||||
def test_update_convergence(self, mock_call, mock_init):
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res',
|
||||
'ResourceWithPropsType')
|
||||
res = generic_rsrc.GenericResource('test_res', tmpl, self.stack)
|
||||
|
|
@ -1552,14 +1573,34 @@ class ResourceTest(common.HeatTestCase):
|
|||
|
||||
res_data = {(1, True): {u'id': 4, u'name': 'A', 'attrs': {}},
|
||||
(2, True): {u'id': 3, u'name': 'B', 'attrs': {}}}
|
||||
res.update_convergence(new_temp.id, res_data, 'engine-007')
|
||||
res.update_convergence(new_temp.id, res_data, 'engine-007', 120)
|
||||
|
||||
mock_update.assert_called_once_with(
|
||||
new_temp.resource_definitions(self.stack)[res.name])
|
||||
expected_rsrc_def = new_temp.resource_definitions(self.stack)[res.name]
|
||||
mock_init.assert_called_once_with(res.update, expected_rsrc_def)
|
||||
mock_call.assert_called_once_with(timeout=120)
|
||||
self.assertEqual(new_temp.id, res.current_template_id)
|
||||
self.assertItemsEqual([3, 4], res.requires)
|
||||
self._assert_resource_lock(res.id, None, 2)
|
||||
|
||||
def test_update_convergence_throws_timeout(self):
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res',
|
||||
'ResourceWithPropsType')
|
||||
res = generic_rsrc.GenericResource('test_res', tmpl, self.stack)
|
||||
res._store()
|
||||
|
||||
new_temp = template.Template({
|
||||
'HeatTemplateFormatVersion': '2012-12-12',
|
||||
'Resources': {
|
||||
'test_res': {'Type': 'ResourceWithPropsType',
|
||||
'Properties': {'Foo': 'abc'}}
|
||||
}}, env=self.env)
|
||||
new_temp.store()
|
||||
|
||||
res_data = {}
|
||||
self.assertRaises(scheduler.Timeout, res.update_convergence,
|
||||
new_temp.id, res_data, 'engine-007',
|
||||
0)
|
||||
|
||||
def test_update_in_progress_convergence(self):
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
|
||||
res = generic_rsrc.GenericResource('test_res', tmpl, self.stack)
|
||||
|
|
@ -1574,14 +1615,18 @@ class ResourceTest(common.HeatTestCase):
|
|||
ex = self.assertRaises(resource.UpdateInProgress,
|
||||
res.update_convergence,
|
||||
'template_key',
|
||||
res_data, 'engine-007')
|
||||
res_data, 'engine-007',
|
||||
self.dummy_timeout)
|
||||
msg = ("The resource %s is already being updated." %
|
||||
res.name)
|
||||
self.assertEqual(msg, six.text_type(ex))
|
||||
# ensure requirements are not updated for failed resource
|
||||
self.assertEqual([1, 2], res.requires)
|
||||
|
||||
def test_delete_convergence_ok(self):
|
||||
@mock.patch.object(resource.scheduler.TaskRunner, '__init__',
|
||||
return_value=None)
|
||||
@mock.patch.object(resource.scheduler.TaskRunner, '__call__')
|
||||
def test_delete_convergence_ok(self, mock_call, mock_init):
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
|
||||
res = generic_rsrc.GenericResource('test_res', tmpl, self.stack)
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
|
||||
|
|
@ -1590,15 +1635,13 @@ class ResourceTest(common.HeatTestCase):
|
|||
res.status = res.COMPLETE
|
||||
res.action = res.CREATE
|
||||
res._store()
|
||||
res_id = res.id
|
||||
res.handle_delete = mock.Mock(return_value=None)
|
||||
res._update_replacement_data = mock.Mock()
|
||||
self._assert_resource_lock(res.id, None, None)
|
||||
res.delete_convergence(2, {}, 'engine-007')
|
||||
self.assertTrue(res.handle_delete.called)
|
||||
self.assertRaises(exception.NotFound,
|
||||
resource_objects.Resource.get_obj,
|
||||
self.stack.context, res_id)
|
||||
res.delete_convergence(2, {}, 'engine-007', 20)
|
||||
|
||||
mock_init.assert_called_once_with(res.destroy)
|
||||
mock_call.assert_called_once_with(timeout=20)
|
||||
self.assertTrue(res._update_replacement_data.called)
|
||||
|
||||
def test_delete_convergence_does_not_delete_same_template_resource(self):
|
||||
|
|
@ -1607,7 +1650,8 @@ class ResourceTest(common.HeatTestCase):
|
|||
res.current_template_id = 'same-template'
|
||||
res._store()
|
||||
res.destroy = mock.Mock()
|
||||
res.delete_convergence('same-template', {}, 'engine-007')
|
||||
res.delete_convergence('same-template', {}, 'engine-007',
|
||||
self.dummy_timeout)
|
||||
self.assertFalse(res.destroy.called)
|
||||
|
||||
def test_delete_convergence_fail(self):
|
||||
|
|
@ -1621,7 +1665,8 @@ class ResourceTest(common.HeatTestCase):
|
|||
res.handle_delete = mock.Mock(side_effect=ValueError('test'))
|
||||
self._assert_resource_lock(res.id, None, None)
|
||||
self.assertRaises(exception.ResourceFailure,
|
||||
res.delete_convergence, 2, {}, 'engine-007')
|
||||
res.delete_convergence, 2, {}, 'engine-007',
|
||||
self.dummy_timeout)
|
||||
self.assertTrue(res.handle_delete.called)
|
||||
|
||||
# confirm that the DB object still exists, and it's lock is released.
|
||||
|
|
@ -1642,7 +1687,7 @@ class ResourceTest(common.HeatTestCase):
|
|||
self._assert_resource_lock(res.id, 'not-this', None)
|
||||
ex = self.assertRaises(resource.UpdateInProgress,
|
||||
res.delete_convergence,
|
||||
1, {}, 'engine-007')
|
||||
1, {}, 'engine-007', self.dummy_timeout)
|
||||
msg = ("The resource %s is already being updated." %
|
||||
res.name)
|
||||
self.assertEqual(msg, six.text_type(ex))
|
||||
|
|
@ -1657,7 +1702,7 @@ class ResourceTest(common.HeatTestCase):
|
|||
res.destroy = mock.Mock()
|
||||
input_data = {(1, False): 4, (2, False): 5} # needed_by resource ids
|
||||
self._assert_resource_lock(res.id, None, None)
|
||||
res.delete_convergence(1, input_data, 'engine-007')
|
||||
res.delete_convergence(1, input_data, 'engine-007', self.dummy_timeout)
|
||||
self.assertItemsEqual([4, 5], res.needed_by)
|
||||
|
||||
@mock.patch.object(resource_objects.Resource, 'get_obj')
|
||||
|
|
@ -1765,6 +1810,14 @@ class ResourceTest(common.HeatTestCase):
|
|||
test_obj.get.side_effect = AttributeError
|
||||
self.assertIsNone(res._show_resource())
|
||||
|
||||
def test_delete_convergence_throws_timeout(self):
|
||||
tmpl = rsrc_defn.ResourceDefinition('test_res', 'Foo')
|
||||
res = generic_rsrc.GenericResource('test_res', tmpl, self.stack)
|
||||
res._store()
|
||||
timeout = 0 # to emulate timeout
|
||||
self.assertRaises(scheduler.Timeout, res.delete_convergence,
|
||||
1, {}, 'engine-007', timeout)
|
||||
|
||||
|
||||
class ResourceAdoptTest(common.HeatTestCase):
|
||||
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
|
||||
import collections
|
||||
import copy
|
||||
import datetime
|
||||
import json
|
||||
import time
|
||||
|
||||
|
|
@ -100,6 +101,59 @@ class StackTest(common.HeatTestCase):
|
|||
timeout_mins=10)
|
||||
self.assertEqual(600, self.stack.timeout_secs())
|
||||
|
||||
@mock.patch.object(stack, 'datetime')
|
||||
def test_time_elapsed(self, mock_dt):
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl)
|
||||
# dummy create time 10:00:00
|
||||
self.stack.created_time = datetime.datetime(2015, 7, 27, 10, 0, 0)
|
||||
# mock utcnow set to 10:10:00 (600s offset)
|
||||
mock_dt.datetime.utcnow.return_value = datetime.datetime(2015, 7, 27,
|
||||
10, 10, 0)
|
||||
self.assertEqual(600, self.stack.time_elapsed())
|
||||
|
||||
@mock.patch.object(stack, 'datetime')
|
||||
def test_time_elapsed_with_updated_time(self, mock_dt):
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl)
|
||||
# dummy create time 10:00:00
|
||||
self.stack.created_time = datetime.datetime(2015, 7, 27, 10, 0, 0)
|
||||
# dummy updated time 11:00:00; should consider this not created_time
|
||||
self.stack.updated_time = datetime.datetime(2015, 7, 27, 11, 0, 0)
|
||||
# mock utcnow set to 11:10:00 (600s offset)
|
||||
mock_dt.datetime.utcnow.return_value = datetime.datetime(2015, 7, 27,
|
||||
11, 10, 0)
|
||||
self.assertEqual(600, self.stack.time_elapsed())
|
||||
|
||||
@mock.patch.object(stack.Stack, 'time_elapsed')
|
||||
def test_time_remaining(self, mock_te):
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl)
|
||||
# mock time elapsed; set to 600 seconds
|
||||
mock_te.return_value = 600
|
||||
# default stack timeout is 3600 seconds; remaining time 3000 secs
|
||||
self.assertEqual(3000, self.stack.time_remaining())
|
||||
|
||||
@mock.patch.object(stack.Stack, 'time_elapsed')
|
||||
def test_has_timed_out(self, mock_te):
|
||||
self.stack = stack.Stack(self.ctx, 'test_stack', self.tmpl)
|
||||
self.stack.status = self.stack.IN_PROGRESS
|
||||
|
||||
# test with timed out stack
|
||||
mock_te.return_value = 3601
|
||||
# default stack timeout is 3600 seconds; stack should time out
|
||||
self.assertTrue(self.stack.has_timed_out())
|
||||
|
||||
# mock time elapsed; set to 600 seconds
|
||||
mock_te.return_value = 600
|
||||
# default stack timeout is 3600 seconds; remaining time 3000 secs
|
||||
self.assertFalse(self.stack.has_timed_out())
|
||||
|
||||
# has_timed_out has no meaning when stack completes/fails;
|
||||
# should return false
|
||||
self.stack.status = self.stack.COMPLETE
|
||||
self.assertFalse(self.stack.has_timed_out())
|
||||
|
||||
self.stack.status = self.stack.FAILED
|
||||
self.assertFalse(self.stack.has_timed_out())
|
||||
|
||||
def test_no_auth_token(self):
|
||||
ctx = utils.dummy_context()
|
||||
ctx.auth_token = None
|
||||
|
|
|
|||
Loading…
Reference in New Issue