openstack/nova
Code Issues Proposed changes

Add RevertResizeTask

Browse Source 
This adds the RevertResizeTask which will be used to orchestrate
the revert of a cross-cell resize. It is responsible for updating
the instance mapping and hidden fields to hide the target cell
instance and show the source cell instance, clean up the target
host / cell, destroy the target cell instance, and (re)spawn the
instance in the source cell.

Part of blueprint cross-cell-resize

Change-Id: I7069f8807f353fa63c96494395bbbf7cc44562fb
This commit is contained in:
Matt Riedemann
2019-02-19 14:51:37 -05:00
parent 750aef54b1
commit f6f4657e9c
2 changed files with 762 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

364
nova/conductor/tasks/cross_cell_migrate.py
View File

@@ -904,7 +904,10 @@ def get_instance_from_source_cell(
target cell database
:param source_compute: name of the source compute service host
:param instance_uuid: UUID of the instance
:returns: Instance object from the source cell database.
:returns: 2-item tuple of:
- Instance object from the source cell database.
- CellMapping object of the source cell mapping
"""
# We can get the source cell via the host mapping based on the
# source_compute in the migration object.
@@ -918,9 +921,10 @@ def get_instance_from_source_cell(
# Now get the instance from the source cell DB using the source
# cell context which will make the source cell instance permanently
# targeted to the source cell database.
return objects.Instance.get_by_uuid(
instance = objects.Instance.get_by_uuid(
source_cell_context, instance_uuid,
expected_attrs=['flavor', 'info_cache', 'system_metadata'])
return instance, source_cell_mapping
class ConfirmResizeTask(base.TaskBase):
@@ -1059,7 +1063,7 @@ class ConfirmResizeTask(base.TaskBase):
def _execute(self):
# First get the instance from the source cell so we can cleanup.
source_cell_instance = get_instance_from_source_cell(
self.context, self.migration.source_compute, self.instance.uuid)
self.context, self.migration.source_compute, self.instance.uuid)[0]
# Send the resize.confirm.start notification(s) using the source
# cell instance since we start there.
self._send_resize_confirm_notification(
@@ -1103,3 +1107,357 @@ class ConfirmResizeTask(base.TaskBase):
scheduler_utils.set_vm_state_and_notify(
self.context, self.instance.uuid, 'compute_task',
'migrate_server', updates, ex, request_spec)
class RevertResizeTask(base.TaskBase):
"""Task to orchestrate a cross-cell resize revert operation.
This task is responsible for coordinating the cleanup of the resources
in the target cell and restoring the server and its related resources
(e.g. networking and volumes) in the source cell.
Upon successful completion the instance mapping should point back at the
source cell, the source cell instance should no longer be hidden and the
instance in the target cell should be destroyed.
"""
def __init__(self, context, instance, migration, legacy_notifier,
compute_rpcapi):
"""Initialize this RevertResizeTask instance
:param context: nova auth request context targeted at the target cell
:param instance: Instance object in "resized" status from the target
cell with task_state "resize_reverting"
:param migration: Migration object from the target cell for the resize
operation expected to have status "reverting"
:param legacy_notifier: LegacyValidatingNotifier for sending legacy
unversioned notifications
:param compute_rpcapi: instance of nova.compute.rpcapi.ComputeAPI
"""
super(RevertResizeTask, self).__init__(context, instance)
self.migration = migration
self.legacy_notifier = legacy_notifier
self.compute_rpcapi = compute_rpcapi
self._source_cell_migration = None
self.volume_api = cinder.API()
def _send_resize_revert_notification(self, instance, phase):
"""Sends an unversioned and versioned resize.revert.(phase)
notification.
:param instance: The instance whose resize is being reverted.
:param phase: The phase for the resize.revert operation (either
"start" or "end").
"""
ctxt = instance._context
# Send the legacy unversioned notification.
compute_utils.notify_about_instance_usage(
self.legacy_notifier, ctxt, instance, 'resize.revert.%s' % phase)
# Send the versioned notification.
compute_utils.notify_about_instance_action(
ctxt, instance, instance.host, # TODO(mriedem): Use CONF.host?
action=fields.NotificationAction.RESIZE_REVERT,
phase=phase)
@staticmethod
def _update_source_obj_from_target_cell(source_obj, target_obj):
"""Updates the object from the source cell using the target cell object
All fields on the source object are updated from the target object
except for the ``id`` and ``created_at`` fields since those value must
not change during an update. The ``updated_at`` field is also skipped
because saving changes to ``source_obj`` will automatically update the
``updated_at`` field.
It is expected that the two objects represent the same thing but from
different cell databases, so for example, a uuid field (if one exists)
should not change.
Note that the changes to ``source_obj`` are not persisted in this
method.
:param source_obj: Versioned object from the source cell database
:param target_obj: Versioned object from the target cell database
"""
ignore_fields = ['created_at', 'id', 'updated_at']
for field in source_obj.obj_fields:
if field in target_obj and field not in ignore_fields:
setattr(source_obj, field, getattr(target_obj, field))
def _update_bdms_in_source_cell(self, source_cell_context):
"""Update BlockDeviceMapppings in the source cell database.
It is possible to attach/detach volumes to/from a resized instance,
which would create/delete BDM records in the target cell, so we have
to recreate newly attached BDMs in the source cell database and
delete any old BDMs that were detached while resized in the target
cell.
:param source_cell_context: nova auth request context targeted at the
source cell database
"""
# TODO(mriedem): Need functional test wrinkle for this. Attach volume2
# while resized, detach volume1 while resized, and make sure those are
# the same when the revert is done.
bdms_from_source_cell = (
objects.BlockDeviceMappingList.get_by_instance_uuid(
source_cell_context, self.instance.uuid))
source_cell_bdms_by_uuid = {
bdm.uuid: bdm for bdm in bdms_from_source_cell}
bdms_from_target_cell = (
objects.BlockDeviceMappingList.get_by_instance_uuid(
self.context, self.instance.uuid))
# Copy new/updated BDMs from the target cell DB to the source cell DB.
for bdm in bdms_from_target_cell:
if bdm.uuid in source_cell_bdms_by_uuid:
# Remove this BDM from the list since we want to preserve it
# along with its attachment_id.
source_cell_bdms_by_uuid.pop(bdm.uuid)
else:
# Newly attached BDM while in the target cell, so create it
# in the source cell.
source_bdm = clone_creatable_object(source_cell_context, bdm)
# revert_snapshot_based_resize_at_dest is going to delete the
# attachment for this BDM so we need to create a new empty
# attachment to reserve this volume so that
# finish_revert_snapshot_based_resize_at_source can use it.
attach_ref = self.volume_api.attachment_create(
source_cell_context, bdm.volume_id, self.instance.uuid)
source_bdm.attachment_id = attach_ref['id']
LOG.debug('Creating BlockDeviceMapping with volume ID %s '
'and attachment %s in the source cell database '
'since the volume was attached while the server was '
'resized.', bdm.volume_id, attach_ref['id'],
instance=self.instance)
source_bdm.create()
# If there are any source bdms left that were not processed from the
# target cell bdms, it means those source bdms were detached while
# resized in the target cell, and we need to delete them from the
# source cell so they don't re-appear once the revert is complete.
self._delete_orphan_source_cell_bdms(source_cell_bdms_by_uuid.values())
def _delete_orphan_source_cell_bdms(self, source_cell_bdms):
"""Deletes orphaned BDMs and volume attachments from the source cell.
If any volumes were detached while the server was resized into the
target cell they are destroyed here so they do not show up again once
the instance is mapped back to the source cell.
:param source_cell_bdms: Iterator of BlockDeviceMapping objects.
"""
for bdm in source_cell_bdms:
LOG.debug('Destroying BlockDeviceMapping with volume ID %s and '
'attachment ID %s from source cell database during '
'cross-cell resize revert since the volume was detached '
'while the server was resized.', bdm.volume_id,
bdm.attachment_id, instance=self.instance)
# First delete the (empty) attachment, created by
# prep_snapshot_based_resize_at_source, so it is not leaked.
try:
self.volume_api.attachment_delete(
bdm._context, bdm.attachment_id)
except Exception as e:
LOG.error('Failed to delete attachment %s for volume %s. The '
'attachment may be leaked and needs to be manually '
'cleaned up. Error: %s', bdm.attachment_id,
bdm.volume_id, e, instance=self.instance)
bdm.destroy()
def _update_instance_actions_in_source_cell(self, source_cell_context):
"""Update instance action records in the source cell database
We need to copy the REVERT_RESIZE instance action and related events
from the target cell to the source cell. Otherwise the revert operation
in the source compute service will not be able to lookup the correct
instance action to track events.
:param source_cell_context: nova auth request context targeted at the
source cell database
"""
# FIXME(mriedem): This is a hack to just get revert working on
# the source; we need to re-create any actions created in the target
# cell DB after the instance was moved while it was in
# VERIFY_RESIZE status, like if volumes were attached/detached.
# Can we use a changes-since filter for that, i.e. find the last
# instance action for the instance in the source cell database and then
# get all instance actions from the target cell database that were
# created after that time.
action = objects.InstanceAction.get_by_request_id(
self.context, self.instance.uuid, self.context.request_id)
new_action = clone_creatable_object(source_cell_context, action)
new_action.create()
# Also create the events under this action.
events = objects.InstanceActionEventList.get_by_action(
self.context, action.id)
for event in events:
new_event = clone_creatable_object(source_cell_context, event)
new_event.create(action.instance_uuid, action.request_id)
def _update_migration_in_source_cell(self, source_cell_context):
"""Update the migration record in the source cell database.
Updates the migration record in the source cell database based on the
current information about the migration in the target cell database.
:param source_cell_context: nova auth request context targeted at the
source cell database
:return: Migration object of the updated source cell database migration
record
"""
source_cell_migration = objects.Migration.get_by_uuid(
source_cell_context, self.migration.uuid)
# The only change we really expect here is the status changing to
# "reverting".
self._update_source_obj_from_target_cell(
source_cell_migration, self.migration)
source_cell_migration.save()
return source_cell_migration
def _update_instance_in_source_cell(self, instance):
"""Updates the instance and related records in the source cell DB.
Before reverting in the source cell we need to copy the
latest state information from the target cell database where the
instance lived before the revert. This is because data about the
instance could have changed while it was in VERIFY_RESIZE status, like
attached volumes.
:param instance: Instance object from the source cell database
:return: Migration object of the updated source cell database migration
record
"""
LOG.debug('Updating instance-related records in the source cell '
'database based on target cell database information.',
instance=instance)
# Copy information from the target cell instance that we need in the
# source cell instance for doing the revert on the source compute host.
instance.system_metadata['old_vm_state'] = (
self.instance.system_metadata.get('old_vm_state'))
instance.old_flavor = instance.flavor
instance.task_state = task_states.RESIZE_REVERTING
instance.save()
source_cell_context = instance._context
self._update_bdms_in_source_cell(source_cell_context)
self._update_instance_actions_in_source_cell(source_cell_context)
source_cell_migration = self._update_migration_in_source_cell(
source_cell_context)
# NOTE(mriedem): We do not have to worry about ports changing while
# resized since the API does not allow attach/detach interface while
# resized. Same for tags.
return source_cell_migration
def _update_instance_mapping(
self, source_cell_instance, source_cell_mapping):
"""Swaps the hidden field value on the source and target cell instance
and updates the instance mapping to point at the source cell.
:param source_cell_instance: Instance object from the source cell DB
:param source_cell_mapping: CellMapping object for the source cell
"""
LOG.debug('Marking instance in target cell as hidden and updating '
'instance mapping to point at source cell %s.',
source_cell_mapping.identity, instance=source_cell_instance)
# Get the instance mapping first to make the window of time where both
# instances are hidden=False as small as possible.
instance_mapping = objects.InstanceMapping.get_by_instance_uuid(
self.context, self.instance.uuid)
# Mark the source cell instance record as hidden=False so it will show
# up when listing servers. Note that because of how the API filters
# duplicate instance records, even if the user is listing servers at
# this exact moment only one copy of the instance will be returned.
source_cell_instance.hidden = False
source_cell_instance.save()
# Update the instance mapping to point at the source cell. We do this
# before cleaning up the target host/cell because that is really best
# effort and if something fails on the target we want the user to
# now interact with the instance in the source cell with the original
# flavor because they are ultimately trying to revert and get back
# there, so if they hard reboot/rebuild after an error (for example)
# that should happen in the source cell.
instance_mapping.cell_mapping = source_cell_mapping
instance_mapping.save()
# Mark the target cell instance record as hidden=True to hide it from
# the user when listing servers.
self.instance.hidden = True
self.instance.save()
def _execute(self):
# Send the resize.revert.start notification(s) using the target
# cell instance since we start there.
self._send_resize_revert_notification(
self.instance, fields.NotificationPhase.START)
source_cell_instance, source_cell_mapping = (
get_instance_from_source_cell(
self.context, self.migration.source_compute,
self.instance.uuid))
# Update the source cell database information based on the target cell
# database, i.e. the instance/migration/BDMs/action records. Do all of
# this before updating the instance mapping in case it fails.
source_cell_migration = self._update_instance_in_source_cell(
source_cell_instance)
# Swap the instance.hidden values and update the instance mapping to
# point at the source cell. From here on out the user will see and
# operate on the instance in the source cell.
self._update_instance_mapping(
source_cell_instance, source_cell_mapping)
# Save off the source cell migration record for rollbacks.
self._source_cell_migration = source_cell_migration
# Clean the instance from the target host.
LOG.debug('Calling destination host %s to revert cross-cell resize.',
self.migration.dest_compute, instance=self.instance)
# Use the EventReport context manager to create the same event that
# the dest compute will create but in the source cell DB so we do not
# have to explicitly copy it over from target to source DB.
event_name = 'compute_revert_snapshot_based_resize_at_dest'
with compute_utils.EventReporter(
source_cell_instance._context, event_name,
self.migration.dest_compute, self.instance.uuid):
self.compute_rpcapi.revert_snapshot_based_resize_at_dest(
self.context, self.instance, self.migration)
# NOTE(mriedem): revert_snapshot_based_resize_at_dest updates the
# target cell instance so if we need to do something with it here
# in the future before destroying it, it should be refreshed.
# Destroy the instance and its related records from the target cell DB.
LOG.info('Deleting instance record from target cell %s',
self.context.cell_uuid, instance=source_cell_instance)
# This needs to be a hard delete because if we retry the resize to the
# target cell we could hit a duplicate entry unique constraint error.
self.instance.destroy(hard_delete=True)
# Launch the guest at the source host with the old flavor.
LOG.debug('Calling source host %s to finish reverting cross-cell '
'resize.', self.migration.source_compute,
instance=self.instance)
self.compute_rpcapi.finish_revert_snapshot_based_resize_at_source(
source_cell_instance._context, source_cell_instance,
source_cell_migration)
# finish_revert_snapshot_based_resize_at_source updates the source cell
# instance so refresh it here so we have the latest copy.
source_cell_instance.refresh()
# Send the resize.revert.end notification using the instance from
# the source cell since we end there.
self._send_resize_revert_notification(
source_cell_instance, fields.NotificationPhase.END)
def rollback(self, ex):
# If we have updated the instance mapping to point at the source
# cell we update the migration from the source cell, otherwise we
# update the migration in the target cell.
migration = self._source_cell_migration \
if self._source_cell_migration else self.migration
migration.status = 'error'
migration.save()
# TODO(mriedem): Will have to think about setting the instance to
# ERROR status and/or recording a fault and sending an
# error notification (create new resize_revert.error notification?).

431
nova/tests/unit/conductor/tasks/test_cross_cell_migrate.py
View File

@@ -15,6 +15,7 @@ import copy
import mock
from oslo_messaging import exceptions as messaging_exceptions
from oslo_utils.fixture import uuidsentinel as uuids
from oslo_utils import timeutils
import six
from nova.compute import instance_actions
@@ -45,8 +46,29 @@ from nova.tests.unit.objects import test_service
from nova.tests.unit.objects import test_vcpu_model
class TargetDBSetupTaskTestCase(
test.TestCase, test_db_api.ModelsObjectComparatorMixin):
class ObjectComparatorMixin(test_db_api.ModelsObjectComparatorMixin):
"""Mixin class to aid in comparing two objects."""
def _compare_objs(self, obj1, obj2, ignored_keys=None):
# We can always ignore id since it is not deterministic when records
# are copied over to the target cell database.
if ignored_keys is None:
ignored_keys = []
if 'id' not in ignored_keys:
ignored_keys.append('id')
prim1 = obj1.obj_to_primitive()['nova_object.data']
prim2 = obj2.obj_to_primitive()['nova_object.data']
if isinstance(obj1, obj_base.ObjectListBase):
self.assertEqual(len(obj1), len(obj2))
prim1 = [o['nova_object.data'] for o in prim1['objects']]
prim2 = [o['nova_object.data'] for o in prim2['objects']]
self._assertEqualListsOfObjects(
prim1, prim2, ignored_keys=ignored_keys)
else:
self._assertEqualObjects(prim1, prim2, ignored_keys=ignored_keys)
class TargetDBSetupTaskTestCase(test.TestCase, ObjectComparatorMixin):
def setUp(self):
super(TargetDBSetupTaskTestCase, self).setUp()
@@ -203,24 +225,6 @@ class TargetDBSetupTaskTestCase(
self.source_context, inst.uuid, expected_attrs=expected_attrs)
return inst, migration
def _compare_objs(self, obj1, obj2, ignored_keys=None):
# We can always ignore id since it is not deterministic when records
# are copied over to the target cell database.
if ignored_keys is None:
ignored_keys = []
if 'id' not in ignored_keys:
ignored_keys.append('id')
prim1 = obj1.obj_to_primitive()['nova_object.data']
prim2 = obj2.obj_to_primitive()['nova_object.data']
if isinstance(obj1, obj_base.ObjectListBase):
self.assertEqual(len(obj1), len(obj2))
prim1 = [o['nova_object.data'] for o in prim1['objects']]
prim2 = [o['nova_object.data'] for o in prim2['objects']]
self._assertEqualListsOfObjects(
prim1, prim2, ignored_keys=ignored_keys)
else:
self._assertEqualObjects(prim1, prim2, ignored_keys=ignored_keys)
def test_execute_and_rollback(self):
"""Happy path test which creates an instance with related records
in a source cell and then executes TargetDBSetupTask to create those
@@ -1117,9 +1121,10 @@ class UtilityTestCase(test.NoDBTestCase):
self.assertEqual(uuids.cell, ctxt.cell_uuid)
return mock.sentinel.instance
mock_get_inst.side_effect = stub_get_by_uuid
inst = cross_cell_migrate.get_instance_from_source_cell(
inst, cell_mapping = cross_cell_migrate.get_instance_from_source_cell(
target_cell_context, 'source-host', uuids.instance)
self.assertIs(inst, mock.sentinel.instance)
self.assertIs(cell_mapping, mock_get_by_host.return_value.cell_mapping)
mock_get_by_host.assert_called_once_with(
target_cell_context, 'source-host')
mock_get_inst.assert_called_once_with(
@@ -1147,11 +1152,13 @@ class ConfirmResizeTaskTestCase(test.NoDBTestCase):
compute_rpcapi)
@mock.patch('nova.conductor.tasks.cross_cell_migrate.'
'get_instance_from_source_cell',
return_value=objects.Instance(
mock.MagicMock(), uuid=uuids.instance))
'get_instance_from_source_cell')
def test_execute(self, mock_get_instance):
mock_get_instance.return_value.destroy = mock.Mock()
source_cell_instance = objects.Instance(
mock.MagicMock(), uuid=uuids.instance)
source_cell_instance.destroy = mock.Mock()
mock_get_instance.return_value = (
source_cell_instance, objects.CellMapping())
with test.nested(
mock.patch.object(self.task, '_send_resize_confirm_notification'),
mock.patch.object(self.task, '_cleanup_source_host'),
@@ -1166,12 +1173,10 @@ class ConfirmResizeTaskTestCase(test.NoDBTestCase):
self.task.instance.uuid)
self.assertEqual(2, _send_resize_confirm_notification.call_count)
_send_resize_confirm_notification.assert_has_calls([
mock.call(mock_get_instance.return_value,
fields.NotificationPhase.START),
mock.call(source_cell_instance, fields.NotificationPhase.START),
mock.call(self.task.instance, fields.NotificationPhase.END)])
_cleanup_source_host.assert_called_once_with(
mock_get_instance.return_value)
mock_get_instance.return_value.destroy.assert_called_once_with(
_cleanup_source_host.assert_called_once_with(source_cell_instance)
source_cell_instance.destroy.assert_called_once_with(
hard_delete=True)
_finish_confirm_in_target_cell.assert_called_once_with()
@@ -1270,3 +1275,369 @@ class ConfirmResizeTaskTestCase(test.NoDBTestCase):
self.task._set_vm_and_task_state()
self.assertEqual(vm_states.ACTIVE, self.task.instance.vm_state)
self.assertIsNone(self.task.instance.task_state)
class RevertResizeTaskTestCase(test.NoDBTestCase, ObjectComparatorMixin):
def setUp(self):
super(RevertResizeTaskTestCase, self).setUp()
target_cell_context = nova_context.get_admin_context()
target_cell_context.cell_uuid = uuids.target_cell
instance = fake_instance.fake_instance_obj(
target_cell_context, **{
'vm_state': vm_states.RESIZED,
'task_state': task_states.RESIZE_REVERTING,
'expected_attrs': ['system_metadata', 'flavor']
})
migration = objects.Migration(
target_cell_context, uuid=uuids.migration, status='reverting',
source_compute='source-host', dest_compute='dest-host')
legacy_notifier = mock.MagicMock()
compute_rpcapi = mock.MagicMock()
self.task = cross_cell_migrate.RevertResizeTask(
target_cell_context, instance, migration, legacy_notifier,
compute_rpcapi)
def _generate_source_cell_instance(self):
source_cell_context = nova_context.get_admin_context()
source_cell_context.cell_uuid = uuids.source_cell
source_cell_instance = self.task.instance.obj_clone()
source_cell_instance._context = source_cell_context
return source_cell_instance
@mock.patch('nova.conductor.tasks.cross_cell_migrate.'
'get_instance_from_source_cell')
@mock.patch('nova.objects.InstanceActionEvent') # Stub EventReport calls.
def test_execute(self, mock_action_event, mock_get_instance):
"""Happy path test for the execute method."""
# Setup mocks.
source_cell_instance = self._generate_source_cell_instance()
source_cell_context = source_cell_instance._context
source_cell_mapping = objects.CellMapping(source_cell_context,
uuid=uuids.source_cell)
mock_get_instance.return_value = (source_cell_instance,
source_cell_mapping)
def stub_update_instance_in_source_cell(*args, **kwargs):
# Ensure _update_instance_mapping is not called before
# _update_instance_in_source_cell.
_update_instance_mapping.assert_not_called()
return mock.sentinel.source_cell_migration
with test.nested(
mock.patch.object(self.task, '_send_resize_revert_notification'),
mock.patch.object(self.task, '_update_instance_in_source_cell',
side_effect=stub_update_instance_in_source_cell),
mock.patch.object(self.task, '_update_instance_mapping'),
mock.patch.object(self.task.instance, 'destroy'),
mock.patch.object(source_cell_instance, 'refresh'),
) as (
_send_resize_revert_notification, _update_instance_in_source_cell,
_update_instance_mapping, mock_inst_destroy, mock_inst_refresh,
):
# Run the code.
self.task.execute()
# Should have sent a start and end notification.
self.assertEqual(2, _send_resize_revert_notification.call_count,
_send_resize_revert_notification.calls)
_send_resize_revert_notification.assert_has_calls([
mock.call(self.task.instance, fields.NotificationPhase.START),
mock.call(source_cell_instance, fields.NotificationPhase.END),
])
mock_get_instance.assert_called_once_with(
self.task.context, self.task.migration.source_compute,
self.task.instance.uuid)
_update_instance_in_source_cell.assert_called_once_with(
source_cell_instance)
_update_instance_mapping.assert_called_once_with(
source_cell_instance, source_cell_mapping)
# _source_cell_migration should have been set for rollbacks
self.assertIs(self.task._source_cell_migration,
mock.sentinel.source_cell_migration)
# Cleanup at dest host.
self.task.compute_rpcapi.revert_snapshot_based_resize_at_dest.\
assert_called_once_with(self.task.context, self.task.instance,
self.task.migration)
# EventReporter should have been used.
event_name = 'compute_revert_snapshot_based_resize_at_dest'
mock_action_event.event_start.assert_called_once_with(
source_cell_context, source_cell_instance.uuid, event_name,
want_result=False, host=self.task.migration.dest_compute)
mock_action_event.event_finish_with_failure.assert_called_once_with(
source_cell_context, source_cell_instance.uuid, event_name,
exc_val=None, exc_tb=None, want_result=False)
# Destroy the instance in the target cell.
mock_inst_destroy.assert_called_once_with(hard_delete=True)
# Cleanup at source host.
self.task.compute_rpcapi.\
finish_revert_snapshot_based_resize_at_source.\
assert_called_once_with(
source_cell_context, source_cell_instance,
mock.sentinel.source_cell_migration)
# Refresh the source cell instance so we have the latest data.
mock_inst_refresh.assert_called_once_with()
def test_rollback_target_cell(self):
"""Tests the case that we did not update the instance mapping
so we set the target cell migration to error status.
"""
with mock.patch.object(self.task.migration, 'save') as mock_save:
self.task.rollback(test.TestingException('zoinks!'))
self.assertEqual('error', self.task.migration.status)
mock_save.assert_called_once_with()
def test_rollback_source_cell(self):
"""Tests the case that we did update the instance mapping
so we set the source cell migration to error status.
"""
self.task._source_cell_migration = objects.Migration(
status='reverting')
with mock.patch.object(self.task._source_cell_migration,
'save') as mock_save:
self.task.rollback(test.TestingException('jinkies!'))
self.assertEqual('error', self.task._source_cell_migration.status)
mock_save.assert_called_once_with()
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.compute.utils.notify_about_instance_action')
def test_send_resize_revert_notification(self, mock_notify_action,
mock_notify_usage):
instance = self.task.instance
self.task._send_resize_revert_notification(instance, 'foo')
# Assert the legacy notification was sent.
mock_notify_usage.assert_called_once_with(
self.task.legacy_notifier, instance._context, instance,
'resize.revert.foo')
# Assert the versioned notification was sent.
mock_notify_action.assert_called_once_with(
instance._context, instance, instance.host,
action=fields.NotificationAction.RESIZE_REVERT, phase='foo')
def test_update_instance_in_source_cell(self):
# Setup mocks.
source_cell_instance = self._generate_source_cell_instance()
source_cell_instance.task_state = None
self.task.instance.system_metadata = {'old_vm_state': vm_states.ACTIVE}
with test.nested(
mock.patch.object(source_cell_instance, 'save'),
mock.patch.object(self.task, '_update_bdms_in_source_cell'),
mock.patch.object(self.task,
'_update_instance_actions_in_source_cell'),
mock.patch.object(self.task, '_update_migration_in_source_cell')
) as (
mock_inst_save, _update_bdms_in_source_cell,
_update_instance_actions_in_source_cell,
_update_migration_in_source_cell
):
# Run the code.
source_cell_migration = self.task._update_instance_in_source_cell(
source_cell_instance)
# The returned object should be the updated migration object from the
# source cell database.
self.assertIs(source_cell_migration,
_update_migration_in_source_cell.return_value)
# Fields on the source cell instance should have been updated.
self.assertEqual(vm_states.ACTIVE,
source_cell_instance.system_metadata['old_vm_state'])
self.assertIs(source_cell_instance.old_flavor,
source_cell_instance.flavor)
self.assertEqual(task_states.RESIZE_REVERTING,
source_cell_instance.task_state)
mock_inst_save.assert_called_once_with()
_update_bdms_in_source_cell.assert_called_once_with(
source_cell_instance._context)
_update_instance_actions_in_source_cell.assert_called_once_with(
source_cell_instance._context)
_update_migration_in_source_cell.assert_called_once_with(
source_cell_instance._context)
@mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid')
@mock.patch('nova.conductor.tasks.cross_cell_migrate.'
'clone_creatable_object')
def test_update_bdms_in_source_cell(self, mock_clone, mock_get_bdms):
"""Test updating BDMs from the target cell to the source cell."""
source_cell_context = nova_context.get_admin_context()
source_cell_context.cell_uuid = uuids.source_cell
# Setup fake bdms.
bdm1 = objects.BlockDeviceMapping(
source_cell_context, uuid=uuids.bdm1, volume_id='vol1',
attachment_id=uuids.attach1)
bdm2 = objects.BlockDeviceMapping(
source_cell_context, uuid=uuids.bdm2, volume_id='vol2',
attachment_id=uuids.attach2)
source_bdms = objects.BlockDeviceMappingList(objects=[bdm1, bdm2])
# With the target BDMs bdm1 from the source is gone and bdm3 is new
# to simulate bdm1 being detached and bdm3 being attached while the
# instance was in VERIFY_RESIZE status.
bdm3 = objects.BlockDeviceMapping(
self.task.context, uuid=uuids.bdm3, volume_id='vol3',
attachment_id=uuids.attach1)
target_bdms = objects.BlockDeviceMappingList(objects=[bdm2, bdm3])
def stub_get_bdms(ctxt, *args, **kwargs):
if ctxt.cell_uuid == uuids.source_cell:
return source_bdms
return target_bdms
mock_get_bdms.side_effect = stub_get_bdms
def stub_mock_clone(ctxt, obj, *args, **kwargs):
# We want to make assertions on our mocks so do not create a copy.
return obj
mock_clone.side_effect = stub_mock_clone
with test.nested(
mock.patch.object(self.task.volume_api, 'attachment_create',
return_value={'id': uuids.attachment_id}),
mock.patch.object(self.task.volume_api, 'attachment_delete'),
mock.patch.object(bdm3, 'create'),
mock.patch.object(bdm1, 'destroy')
) as (
mock_attachment_create, mock_attachment_delete,
mock_bdm_create, mock_bdm_destroy
):
self.task._update_bdms_in_source_cell(source_cell_context)
# Should have gotten BDMs from the source and target cell (order does
# not matter).
self.assertEqual(2, mock_get_bdms.call_count, mock_get_bdms.calls)
mock_get_bdms.assert_has_calls([
mock.call(source_cell_context, self.task.instance.uuid),
mock.call(self.task.context, self.task.instance.uuid)],
any_order=True)
# Since bdm3 was new in the target cell an attachment should have been
# created for it in the source cell.
mock_attachment_create.assert_called_once_with(
source_cell_context, bdm3.volume_id, self.task.instance.uuid)
self.assertEqual(uuids.attachment_id, bdm3.attachment_id)
# And bdm3 should have been created in the source cell.
mock_bdm_create.assert_called_once_with()
# Since bdm1 was not in the target cell it should be destroyed in the
# source cell since we can assume it was detached from the target host
# in the target cell while the instance was in VERIFY_RESIZE status.
mock_attachment_delete.assert_called_once_with(
bdm1._context, bdm1.attachment_id)
mock_bdm_destroy.assert_called_once_with()
@mock.patch('nova.objects.BlockDeviceMapping.destroy')
def test_delete_orphan_source_cell_bdms_attach_delete_fails(self, destroy):
"""Tests attachment_delete failing but not being fatal."""
source_cell_context = nova_context.get_admin_context()
bdm1 = objects.BlockDeviceMapping(
source_cell_context, volume_id='vol1', attachment_id=uuids.attach1)
bdm2 = objects.BlockDeviceMapping(
source_cell_context, volume_id='vol2', attachment_id=uuids.attach2)
source_cell_bdms = objects.BlockDeviceMappingList(objects=[bdm1, bdm2])
with mock.patch.object(self.task.volume_api,
'attachment_delete') as attachment_delete:
# First call to attachment_delete fails, second is OK.
attachment_delete.side_effect = [
test.TestingException('cinder is down'), None]
self.task._delete_orphan_source_cell_bdms(source_cell_bdms)
attachment_delete.assert_has_calls([
mock.call(bdm1._context, bdm1.attachment_id),
mock.call(bdm2._context, bdm2.attachment_id)])
self.assertEqual(2, destroy.call_count, destroy.mock_calls)
self.assertIn('cinder is down', self.stdlog.logger.output)
@mock.patch('nova.objects.InstanceAction.get_by_request_id')
@mock.patch('nova.objects.InstanceActionEventList.get_by_action')
@mock.patch('nova.objects.InstanceAction.create')
@mock.patch('nova.objects.InstanceActionEvent.create')
def test_update_instance_actions_in_source_cell(
self, mock_event_create, mock_action_create, mock_get_events,
mock_get_action):
"""Tests copying instance actions from the target to source cell."""
source_cell_context = nova_context.get_admin_context()
source_cell_context.cell_uuid = uuids.source_cell
# Setup a fake action and fake event.
action = objects.InstanceAction(
id=1, action=instance_actions.REVERT_RESIZE,
instance_uuid=self.task.instance.uuid,
request_id=self.task.context.request_id)
mock_get_action.return_value = action
event = objects.InstanceActionEvent(
id=2, action_id=action.id,
event='conductor_revert_snapshot_based_resize')
mock_get_events.return_value = objects.InstanceActionEventList(
objects=[event])
# Run the code.
self.task._update_instance_actions_in_source_cell(source_cell_context)
# Should have created a clone of the action and event.
mock_get_action.assert_called_once_with(
self.task.context, self.task.instance.uuid,
self.task.context.request_id)
mock_get_events.assert_called_once_with(self.task.context, action.id)
mock_action_create.assert_called_once_with()
mock_event_create.assert_called_once_with(
action.instance_uuid, action.request_id)
def test_update_source_obj_from_target_cell(self):
# Create a fake source object to be updated.
t1 = timeutils.utcnow()
source_obj = objects.Migration(id=1, created_at=t1, updated_at=t1,
uuid=uuids.migration,
status='post-migrating')
t2 = timeutils.utcnow()
target_obj = objects.Migration(id=2, created_at=t2, updated_at=t2,
uuid=uuids.migration,
status='reverting',
# Add a field that is not in source_obj.
migration_type='resize')
# Run the copy code.
self.task._update_source_obj_from_target_cell(source_obj, target_obj)
# First make sure that id, created_at and updated_at are not changed.
ignored_keys = ['id', 'created_at', 'updated_at']
for field in ignored_keys:
self.assertNotEqual(getattr(source_obj, field),
getattr(target_obj, field))
# Now make sure the rest of the fields are the same.
self._compare_objs(source_obj, target_obj, ignored_keys=ignored_keys)
@mock.patch('nova.objects.Migration.get_by_uuid')
def test_update_migration_in_source_cell(self, mock_get_migration):
"""Tests updating the migration record in the source cell from the
target cell.
"""
source_cell_context = nova_context.get_admin_context()
with mock.patch.object(
self.task,
'_update_source_obj_from_target_cell') as mock_update_obj:
source_cell_migration = \
self.task._update_migration_in_source_cell(source_cell_context)
mock_get_migration.assert_called_once_with(source_cell_context,
self.task.migration.uuid)
mock_update_obj.assert_called_once_with(source_cell_migration,
self.task.migration)
self.assertIs(source_cell_migration, mock_get_migration.return_value)
source_cell_migration.save.assert_called_once_with()
@mock.patch('nova.objects.InstanceMapping.get_by_instance_uuid')
def test_update_instance_mapping(self, get_inst_map):
"""Tests updating the instance mapping from the target to source cell.
"""
source_cell_instance = self._generate_source_cell_instance()
source_cell_context = source_cell_instance._context
source_cell_mapping = objects.CellMapping(source_cell_context,
uuid=uuids.source_cell)
inst_map = objects.InstanceMapping(
cell_mapping=objects.CellMapping(uuids.target_cell))
get_inst_map.return_value = inst_map
with test.nested(
mock.patch.object(source_cell_instance, 'save'),
mock.patch.object(self.task.instance, 'save'),
mock.patch.object(inst_map, 'save')
) as (
source_inst_save, target_inst_save, inst_map_save
):
self.task._update_instance_mapping(source_cell_instance,
source_cell_mapping)
get_inst_map.assert_called_once_with(self.task.context,
self.task.instance.uuid)
# The source cell instance should not be hidden.
self.assertFalse(source_cell_instance.hidden)
source_inst_save.assert_called_once_with()
# The instance mapping should point at the source cell.
self.assertIs(source_cell_mapping, inst_map.cell_mapping)
inst_map_save.assert_called_once_with()
# The target cell instance should be hidden.
self.assertTrue(self.task.instance.hidden)
target_inst_save.assert_called_once_with()