openstack
/
nova
Code Issues Proposed changes

Allow force-delete irrespective of VM task_state 

Cannot delete vm instance if duplicate delete requests are sent.

When user sends request to delete an instance, its task_state gets
changed to 'deleting' state. When an instance task_state is already in
'deleting' state and if at that moment the rabbitmq-server get crashed
by some reasons, then the instance task_state remains in 'deleting'
state and user won't be able to delete the instance forever.

At this moment, there is only one way to delete the instances, whose
task_state is in 'deleting' state, by restarting the nova-compute
services where these instances are running.

To avoid restarting the nova-compute service manually, modified the
force-delete api to allow instance deletion irrespective of
instance task_state.

Added new module to use delete_types as constants and replaced all
delete_type string occurances with new constants.

Closes-Bug: #1308342

Change-Id: I4d0e47662a80109ef9622d85455587d487e47c01
This commit is contained in:
Rajesh Tailor 2014-09-15 07:20:14 -07:00
parent b16f7e08c5
commit 222d44532c
11 changed files with 223 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
nova/cells/messaging.py
View File

@ -41,6 +41,7 @@ import six
from nova.cells import state as cells_state
from nova.cells import utils as cells_utils
from nova import compute
from nova.compute import delete_types
from nova.compute import rpcapi as compute_rpcapi
from nova.compute import task_states
from nova.compute import vm_states
@ -843,7 +844,7 @@ class _TargetedMessageMethods(_BaseMessageMethods):
self.msg_runner.instance_destroy_at_top(ctxt,
instance)
except exception.InstanceInfoCacheNotFound:
if method != 'delete':
if method != delete_types.DELETE:
raise
fn = getattr(self.compute_api, method, None)
@ -876,10 +877,12 @@ class _TargetedMessageMethods(_BaseMessageMethods):
return self.host_api.get_host_uptime(message.ctxt, host_name)
def terminate_instance(self, message, instance):
self._call_compute_api_with_obj(message.ctxt, instance, 'delete')
self._call_compute_api_with_obj(message.ctxt, instance,
delete_types.DELETE)
def soft_delete_instance(self, message, instance):
self._call_compute_api_with_obj(message.ctxt, instance, 'soft_delete')
self._call_compute_api_with_obj(message.ctxt, instance,
delete_types.SOFT_DELETE)
def pause_instance(self, message, instance):
"""Pause an instance via compute_api.pause()."""
@ -1095,7 +1098,7 @@ class _BroadcastMessageMethods(_BaseMessageMethods):
"""
LOG.debug("Got broadcast to %(delete_type)s delete instance",
{'delete_type': delete_type}, instance=instance)
if delete_type == 'soft':
if delete_type == delete_types.SOFT_DELETE:
self.compute_api.soft_delete(message.ctxt, instance)
else:
self.compute_api.delete(message.ctxt, instance)

4
nova/cells/rpcapi.py
View File

@ -170,8 +170,8 @@ class CellsAPI(object):
self.client.cast(ctxt, 'instance_destroy_at_top', instance=instance_p)
def instance_delete_everywhere(self, ctxt, instance, delete_type):
"""Delete instance everywhere. delete_type may be 'soft'
or 'hard'. This is generally only used to resolve races
"""Delete instance everywhere. delete_type may be 'soft_delete'
or 'delete'. This is generally only used to resolve races
when API cell doesn't know to what cell an instance belongs.
"""
if not CONF.cells.enable:

20
nova/compute/api.py
View File

@ -35,6 +35,7 @@ import six
from nova import availability_zones
from nova import block_device
from nova.cells import opts as cells_opts
from nova.compute import delete_types
from nova.compute import flavors
from nova.compute import instance_actions
from nova.compute import power_state
@ -1592,8 +1593,10 @@ class API(base.Base):
if self.servicegroup_api.service_is_up(service):
is_up = True
if original_task_state in (task_states.DELETING,
task_states.SOFT_DELETING):
if (delete_type != delete_types.FORCE_DELETE
and original_task_state in (
task_states.DELETING,
task_states.SOFT_DELETING)):
LOG.info(_('Instance is already in deleting state, '
'ignoring this request'), instance=instance)
quotas.rollback()
@ -1781,12 +1784,14 @@ class API(base.Base):
LOG.debug('Going to try to soft delete instance',
instance=instance)
self._delete(context, instance, 'soft_delete', self._do_soft_delete,
self._delete(context, instance, delete_types.SOFT_DELETE,
self._do_soft_delete,
task_state=task_states.SOFT_DELETING,
deleted_at=timeutils.utcnow())
def _delete_instance(self, context, instance):
self._delete(context, instance, 'delete', self._do_delete,
def _delete_instance(self, context, instance,
delete_type=delete_types.DELETE):
self._delete(context, instance, delete_type, self._do_delete,
task_state=task_states.DELETING)
@wrap_check_policy
@ -1830,10 +1835,11 @@ class API(base.Base):
@wrap_check_policy
@check_instance_lock
@check_instance_state(must_have_launched=False)
@check_instance_state(task_state=None,
must_have_launched=False)
def force_delete(self, context, instance):
"""Force delete an instance in any vm_state/task_state."""
self._delete_instance(context, instance)
self._delete_instance(context, instance, delete_types.FORCE_DELETE)
def force_stop(self, context, instance, do_cast=True):
LOG.debug("Going to try to stop instance", instance=instance)

14
nova/compute/cells_api.py
View File

@ -23,6 +23,7 @@ from nova import block_device
from nova.cells import rpcapi as cells_rpcapi
from nova.cells import utils as cells_utils
from nova.compute import api as compute_api
from nova.compute import delete_types
from nova.compute import rpcapi as compute_rpcapi
from nova import exception
from nova import objects
@ -225,14 +226,13 @@ class ComputeCellsAPI(compute_api.API):
return rv
def soft_delete(self, context, instance):
self._handle_cell_delete(context, instance, 'soft_delete')
self._handle_cell_delete(context, instance, delete_types.SOFT_DELETE)
def delete(self, context, instance):
self._handle_cell_delete(context, instance, 'delete')
self._handle_cell_delete(context, instance, delete_types.DELETE)
def _handle_cell_delete(self, context, instance, method_name):
def _handle_cell_delete(self, context, instance, delete_type):
if not instance['cell_name']:
delete_type = method_name == 'soft_delete' and 'soft' or 'hard'
self.cells_rpcapi.instance_delete_everywhere(context,
instance, delete_type)
bdms = block_device.legacy_mapping(
@ -241,11 +241,11 @@ class ComputeCellsAPI(compute_api.API):
# NOTE(danms): If we try to delete an instance with no cell,
# there isn't anything to salvage, so we can hard-delete here.
super(ComputeCellsAPI, self)._local_delete(context, instance, bdms,
method_name,
delete_type,
self._do_delete)
return
method = getattr(super(ComputeCellsAPI, self), method_name)
method = getattr(super(ComputeCellsAPI, self), delete_type)
method(context, instance)
@check_instance_cell
@ -258,7 +258,7 @@ class ComputeCellsAPI(compute_api.API):
def force_delete(self, context, instance):
"""Force delete a previously deleted (but not reclaimed) instance."""
super(ComputeCellsAPI, self).force_delete(context, instance)
self._cast_to_cells(context, instance, 'force_delete')
self._cast_to_cells(context, instance, delete_types.FORCE_DELETE)
@check_instance_cell
def evacuate(self, context, instance, *args, **kwargs):

26
nova/compute/delete_types.py Normal file
View File

@ -0,0 +1,26 @@
# Copyright 2010 OpenStack Foundation
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Possible delete types for instance deletion.
"""
SOFT_DELETE = 'soft_delete' # Soft delete the instance.
DELETE = 'delete' # Delete the instance.
FORCE_DELETE = 'force_delete' # Force delete the instance.

5
nova/tests/api/openstack/compute/plugins/v3/test_servers.py
View File

@ -41,6 +41,7 @@ from nova.api.openstack.compute.schemas.v3 import servers as servers_schema
from nova.api.openstack.compute import views
from nova.api.openstack import extensions
from nova.compute import api as compute_api
from nova.compute import delete_types
from nova.compute import flavors
from nova.compute import task_states
from nova.compute import vm_states
@ -1346,9 +1347,9 @@ class ServersControllerDeleteTest(ControllerTest):
def test_delete_locked_server(self):
req = self._create_delete_request(FAKE_UUID)
self.stubs.Set(compute_api.API, 'soft_delete',
self.stubs.Set(compute_api.API, delete_types.SOFT_DELETE,
fakes.fake_actions_to_locked_server)
self.stubs.Set(compute_api.API, 'delete',
self.stubs.Set(compute_api.API, delete_types.DELETE,
fakes.fake_actions_to_locked_server)
self.assertRaises(webob.exc.HTTPConflict, self.controller.delete,

5
nova/tests/api/openstack/compute/test_servers.py
View File

@ -39,6 +39,7 @@ from nova.api.openstack.compute import views
from nova.api.openstack import extensions
from nova.api.openstack import xmlutil
from nova.compute import api as compute_api
from nova.compute import delete_types
from nova.compute import flavors
from nova.compute import task_states
from nova.compute import vm_states
@ -1481,9 +1482,9 @@ class ServersControllerDeleteTest(ControllerTest):
def test_delete_locked_server(self):
req = self._create_delete_request(FAKE_UUID)
self.stubs.Set(compute_api.API, 'soft_delete',
self.stubs.Set(compute_api.API, delete_types.SOFT_DELETE,
fakes.fake_actions_to_locked_server)
self.stubs.Set(compute_api.API, 'delete',
self.stubs.Set(compute_api.API, delete_types.DELETE,
fakes.fake_actions_to_locked_server)
self.assertRaises(webob.exc.HTTPConflict, self.controller.delete,

22
nova/tests/cells/test_cells_messaging.py
View File

@ -28,6 +28,7 @@ from oslo.utils import timeutils
from nova.cells import messaging
from nova.cells import utils as cells_utils
from nova.compute import delete_types
from nova.compute import task_states
from nova.compute import vm_states
from nova import context
@ -1297,7 +1298,7 @@ class CellsTargetedMethodsTestCase(test.TestCase):
(), {}, (), {}, False)
def test_soft_delete_instance(self):
self._test_instance_action_method('soft_delete',
self._test_instance_action_method(delete_types.SOFT_DELETE,
(), {}, (), {}, False)
def test_pause_instance(self):
@ -1616,10 +1617,10 @@ class CellsBroadcastMethodsTestCase(test.TestCase):
instance = {'uuid': 'meow'}
# Should not be called in src (API cell)
self.mox.StubOutWithMock(self.src_compute_api, 'delete')
self.mox.StubOutWithMock(self.src_compute_api, delete_types.DELETE)
self.mox.StubOutWithMock(self.mid_compute_api, 'delete')
self.mox.StubOutWithMock(self.tgt_compute_api, 'delete')
self.mox.StubOutWithMock(self.mid_compute_api, delete_types.DELETE)
self.mox.StubOutWithMock(self.tgt_compute_api, delete_types.DELETE)
self.mid_compute_api.delete(self.ctxt, instance)
self.tgt_compute_api.delete(self.ctxt, instance)
@ -1627,7 +1628,7 @@ class CellsBroadcastMethodsTestCase(test.TestCase):
self.mox.ReplayAll()
self.src_msg_runner.instance_delete_everywhere(self.ctxt,
instance, 'hard')
instance, delete_types.DELETE)
def test_instance_soft_delete_everywhere(self):
# Reset this, as this is a broadcast down.
@ -1635,10 +1636,13 @@ class CellsBroadcastMethodsTestCase(test.TestCase):
instance = {'uuid': 'meow'}
# Should not be called in src (API cell)
self.mox.StubOutWithMock(self.src_compute_api, 'soft_delete')
self.mox.StubOutWithMock(self.src_compute_api,
delete_types.SOFT_DELETE)
self.mox.StubOutWithMock(self.mid_compute_api, 'soft_delete')
self.mox.StubOutWithMock(self.tgt_compute_api, 'soft_delete')
self.mox.StubOutWithMock(self.mid_compute_api,
delete_types.SOFT_DELETE)
self.mox.StubOutWithMock(self.tgt_compute_api,
delete_types.SOFT_DELETE)
self.mid_compute_api.soft_delete(self.ctxt, instance)
self.tgt_compute_api.soft_delete(self.ctxt, instance)
@ -1646,7 +1650,7 @@ class CellsBroadcastMethodsTestCase(test.TestCase):
self.mox.ReplayAll()
self.src_msg_runner.instance_delete_everywhere(self.ctxt,
instance, 'soft')
instance, delete_types.SOFT_DELETE)
def test_instance_fault_create_at_top(self):
fake_instance_fault = {'id': 1,

3
nova/tests/compute/test_compute.py
View File

@ -44,6 +44,7 @@ from nova import block_device
from nova import compute
from nova.compute import api as compute_api
from nova.compute import arch
from nova.compute import delete_types
from nova.compute import flavors
from nova.compute import manager as compute_manager
from nova.compute import power_state
@ -4286,7 +4287,7 @@ class ComputeTestCase(BaseTestCase):
def fake_soft_delete(*args, **kwargs):
raise test.TestingException()
self.stubs.Set(self.compute.driver, 'soft_delete',
self.stubs.Set(self.compute.driver, delete_types.SOFT_DELETE,
fake_soft_delete)
resvs = self._ensure_quota_reservations_rolledback(instance)

164
nova/tests/compute/test_compute_api.py
View File

@ -25,6 +25,7 @@ from oslo.utils import timeutils
from nova.compute import api as compute_api
from nova.compute import arch
from nova.compute import cells_api as compute_cells_api
from nova.compute import delete_types
from nova.compute import flavors
from nova.compute import instance_actions
from nova.compute import task_states
@ -578,8 +579,9 @@ class _ComputeAPIUnitTestMixIn(object):
self.context.elevated().AndReturn(self.context)
self.compute_api.network_api.deallocate_for_instance(
self.context, inst)
state = ('soft' in delete_type and vm_states.SOFT_DELETED or
vm_states.DELETED)
state = (delete_types.SOFT_DELETE in delete_type and
vm_states.SOFT_DELETED or
vm_states.DELETED)
updates.update({'vm_state': state,
'task_state': None,
'terminated_at': delete_time})
@ -606,10 +608,10 @@ class _ComputeAPIUnitTestMixIn(object):
delete_time = datetime.datetime(1955, 11, 5, 9, 30,
tzinfo=iso8601.iso8601.Utc())
timeutils.set_time_override(delete_time)
task_state = (delete_type == 'soft_delete' and
task_state = (delete_type == delete_types.SOFT_DELETE and
task_states.SOFT_DELETING or task_states.DELETING)
updates = {'progress': 0, 'task_state': task_state}
if delete_type == 'soft_delete':
if delete_type == delete_types.SOFT_DELETE:
updates['deleted_at'] = delete_time
self.mox.StubOutWithMock(inst, 'save')
self.mox.StubOutWithMock(objects.BlockDeviceMappingList,
@ -698,10 +700,11 @@ class _ComputeAPIUnitTestMixIn(object):
cast_reservations = None
else:
cast_reservations = reservations
if delete_type == 'soft_delete':
if delete_type == delete_types.SOFT_DELETE:
rpcapi.soft_delete_instance(self.context, inst,
reservations=cast_reservations)
elif delete_type in ['delete', 'force_delete']:
elif delete_type in [delete_types.DELETE,
delete_types.FORCE_DELETE]:
rpcapi.terminate_instance(self.context, inst, [],
reservations=cast_reservations)
@ -720,59 +723,180 @@ class _ComputeAPIUnitTestMixIn(object):
self.mox.UnsetStubs()
def test_delete(self):
self._test_delete('delete')
self._test_delete(delete_types.DELETE)
def test_delete_if_not_launched(self):
self._test_delete('delete', launched_at=None)
self._test_delete(delete_types.DELETE, launched_at=None)
def test_delete_in_resizing(self):
self._test_delete('delete', task_state=task_states.RESIZE_FINISH)
self._test_delete(delete_types.DELETE,
task_state=task_states.RESIZE_FINISH)
def test_delete_in_resized(self):
self._test_delete('delete', vm_state=vm_states.RESIZED)
self._test_delete(delete_types.DELETE, vm_state=vm_states.RESIZED)
def test_delete_shelved(self):
fake_sys_meta = {'shelved_image_id': SHELVED_IMAGE}
self._test_delete('delete',
self._test_delete(delete_types.DELETE,
vm_state=vm_states.SHELVED,
system_metadata=fake_sys_meta)
def test_delete_shelved_offloaded(self):
fake_sys_meta = {'shelved_image_id': SHELVED_IMAGE}
self._test_delete('delete',
self._test_delete(delete_types.DELETE,
vm_state=vm_states.SHELVED_OFFLOADED,
system_metadata=fake_sys_meta)
def test_delete_shelved_image_not_found(self):
fake_sys_meta = {'shelved_image_id': SHELVED_IMAGE_NOT_FOUND}
self._test_delete('delete',
self._test_delete(delete_types.DELETE,
vm_state=vm_states.SHELVED_OFFLOADED,
system_metadata=fake_sys_meta)
def test_delete_shelved_image_not_authorized(self):
fake_sys_meta = {'shelved_image_id': SHELVED_IMAGE_NOT_AUTHORIZED}
self._test_delete('delete',
self._test_delete(delete_types.DELETE,
vm_state=vm_states.SHELVED_OFFLOADED,
system_metadata=fake_sys_meta)
def test_delete_shelved_exception(self):
fake_sys_meta = {'shelved_image_id': SHELVED_IMAGE_EXCEPTION}
self._test_delete('delete',
self._test_delete(delete_types.DELETE,
vm_state=vm_states.SHELVED,
system_metadata=fake_sys_meta)
def test_delete_with_down_host(self):
self._test_delete('delete', host='down-host')
self._test_delete(delete_types.DELETE, host='down-host')
def test_delete_soft_with_down_host(self):
self._test_delete('soft_delete', host='down-host')
self._test_delete(delete_types.SOFT_DELETE, host='down-host')
def test_delete_soft(self):
self._test_delete('soft_delete')
self._test_delete(delete_types.SOFT_DELETE)
def test_delete_forced(self):
for vm_state in self._get_vm_states():
self._test_delete('force_delete', vm_state=vm_state)
self._test_delete(delete_types.FORCE_DELETE, vm_state=vm_state)
def test_delete_forced_when_task_state_deleting(self):
for vm_state in self._get_vm_states():
self._test_delete(delete_types.FORCE_DELETE, vm_state=vm_state,
task_state=task_states.DELETING)
def test_no_delete_when_task_state_deleting(self):
if self.cell_type == 'api':
# In 'api' cell, the callback terminate_instance will
# get called, and quota will be committed before returning.
# It doesn't check for below condition, hence skipping the test.
"""
if original_task_state in (task_states.DELETING,
task_states.SOFT_DELETING):
LOG.info(_('Instance is already in deleting state, '
'ignoring this request'), instance=instance)
quotas.rollback()
return
"""
self.skipTest("API cell doesn't delete instance directly.")
attrs = {}
fake_sys_meta = {'shelved_image_id': SHELVED_IMAGE}
for vm_state in self._get_vm_states():
if vm_state in (vm_states.SHELVED, vm_states.SHELVED_OFFLOADED):
attrs.update({'system_metadata': fake_sys_meta})
attrs.update({'vm_state': vm_state, 'task_state': 'deleting'})
reservations = ['fake-resv']
inst = self._create_instance_obj()
inst.update(attrs)
inst._context = self.context
deltas = {'instances': -1,
'cores': -inst.vcpus,
'ram': -inst.memory_mb}
delete_time = datetime.datetime(1955, 11, 5, 9, 30,
tzinfo=iso8601.iso8601.Utc())
timeutils.set_time_override(delete_time)
bdms = []
migration = objects.Migration._from_db_object(
self.context, objects.Migration(),
test_migration.fake_db_migration())
fake_quotas = objects.Quotas.from_reservations(self.context,
['rsvs'])
image_api = self.compute_api.image_api
rpcapi = self.compute_api.compute_rpcapi
with contextlib.nested(
mock.patch.object(image_api, 'delete'),
mock.patch.object(inst, 'save'),
mock.patch.object(objects.BlockDeviceMappingList,
'get_by_instance_uuid',
return_value=bdms),
mock.patch.object(objects.Migration,
'get_by_instance_and_status'),
mock.patch.object(quota.QUOTAS, 'reserve',
return_value=reservations),
mock.patch.object(self.context, 'elevated',
return_value=self.context),
mock.patch.object(db, 'service_get_by_compute_host',
return_value=test_service.fake_service),
mock.patch.object(self.compute_api.servicegroup_api,
'service_is_up',
return_value=inst.host != 'down-host'),
mock.patch.object(self.compute_api,
'_downsize_quota_delta',
return_value=fake_quotas),
mock.patch.object(self.compute_api,
'_reserve_quota_delta'),
mock.patch.object(self.compute_api,
'_record_action_start'),
mock.patch.object(db, 'instance_update_and_get_original'),
mock.patch.object(inst.info_cache, 'delete'),
mock.patch.object(self.compute_api.network_api,
'deallocate_for_instance'),
mock.patch.object(db, 'instance_system_metadata_get'),
mock.patch.object(db, 'instance_destroy'),
mock.patch.object(compute_utils,
'notify_about_instance_usage'),
mock.patch.object(quota.QUOTAS, 'commit'),
mock.patch.object(quota.QUOTAS, 'rollback'),
mock.patch.object(rpcapi, 'confirm_resize'),
mock.patch.object(rpcapi, 'terminate_instance')
) as (
image_delete,
save,
get_by_instance_uuid,
get_by_instance_and_status,
reserve,
elevated,
service_get_by_compute_host,
service_is_up,
_downsize_quota_delta,
_reserve_quota_delta,
_record_action_start,
instance_update_and_get_original,
delete,
deallocate_for_instance,
instance_system_metadata_get,
instance_destroy,
notify_about_instance_usage,
commit,
rollback,
confirm_resize,
terminate_instance
):
if (inst.vm_state in (vm_states.SHELVED,
vm_states.SHELVED_OFFLOADED)):
image_delete.return_value = True
if inst.vm_state == vm_states.RESIZED:
get_by_instance_and_status.return_value = migration
_downsize_quota_delta.return_value = deltas
self.compute_api.delete(self.context, inst)
self.assertEqual(1, rollback.call_count)
self.assertEqual(0, terminate_instance.call_count)
def test_delete_fast_if_host_not_set(self):
inst = self._create_instance_obj()
@ -880,7 +1004,7 @@ class _ComputeAPIUnitTestMixIn(object):
self.mox.ReplayAll()
self.compute_api._local_delete(self.context, inst, bdms,
'delete',
delete_types.DELETE,
_fake_do_delete)
def test_delete_disabled(self):

5
nova/tests/compute/test_compute_cells.py
View File

@ -25,6 +25,7 @@ from oslo.utils import timeutils
from nova.cells import manager
from nova.compute import api as compute_api
from nova.compute import cells_api as compute_cells_api
from nova.compute import delete_types
from nova.compute import flavors
from nova.compute import vm_states
from nova import context
@ -147,7 +148,7 @@ class CellsComputeAPITestCase(test_compute.ComputeAPITestCase):
'instance_delete_everywhere')
inst = self._create_fake_instance_obj()
cells_rpcapi.instance_delete_everywhere(self.context,
inst, 'hard')
inst, delete_types.DELETE)
self.mox.ReplayAll()
self.stubs.Set(self.compute_api.network_api, 'deallocate_for_instance',
lambda *a, **kw: None)
@ -159,7 +160,7 @@ class CellsComputeAPITestCase(test_compute.ComputeAPITestCase):
'instance_delete_everywhere')
inst = self._create_fake_instance_obj()
cells_rpcapi.instance_delete_everywhere(self.context,
inst, 'soft')
inst, delete_types.SOFT_DELETE)
self.mox.ReplayAll()
self.stubs.Set(self.compute_api.network_api, 'deallocate_for_instance',
lambda *a, **kw: None)