openstack
/
nova
Code Issues Proposed changes

Pre-create migration object 

We need to do this so we can have a migration uuid at the time we
call to scheduler to allocate for the new host. This just does the
plumbing through the RPC layers. The compute-side code can already
tolerate a migration having been already created for things like
live migration, so we just have to plumb it through.

Related to blueprint migration-allocations

Change-Id: I6bc6d28655368084f08fed9c4f56a285b7063338
This commit is contained in:
Dan Smith 2017-08-29 14:20:14 -07:00
parent e4f89ed5dd
commit 9868a4d9c1
11 changed files with 216 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
doc/notification_samples/service-update.json
View File

@ -13,7 +13,7 @@
"disabled_reason": null,
"report_count": 1,
"forced_down": false,
"version": 22,
"version": 23,
"availability_zone": null,
"uuid": "fa69c544-906b-4a6a-a9c6-c1f7a8078c73"
}

33
nova/compute/manager.py
View File

@ -117,13 +117,18 @@ def errors_out_migration_ctxt(migration):
yield
except Exception:
with excutils.save_and_reraise_exception():
migration.status = 'error'
try:
with migration.obj_as_admin():
migration.save()
except Exception:
LOG.debug('Error setting migration status for instance %s.',
migration.instance_uuid, exc_info=True)
if migration:
# We may have been passed None for our migration if we're
# receiving from an older client. The migration will be
# errored via the legacy path.
migration.status = 'error'
try:
with migration.obj_as_admin():
migration.save()
except Exception:
LOG.debug(
'Error setting migration status for instance %s.',
migration.instance_uuid, exc_info=True)
@utils.expects_func_args('migration')
@ -481,7 +486,7 @@ class ComputeVirtAPI(virtapi.VirtAPI):
class ComputeManager(manager.Manager):
"""Manages the running instances from creation to destruction."""
target = messaging.Target(version='4.17')
target = messaging.Target(version='4.18')
# How long to wait in seconds before re-issuing a shutdown
# signal to an instance during power off. The overall
@ -3814,7 +3819,7 @@ class ComputeManager(manager.Manager):
context, instance, "resize.revert.end")
def _prep_resize(self, context, image, instance, instance_type,
filter_properties, node, clean_shutdown=True):
filter_properties, node, migration, clean_shutdown=True):
if not filter_properties:
filter_properties = {}
@ -3845,7 +3850,8 @@ class ComputeManager(manager.Manager):
limits = filter_properties.get('limits', {})
rt = self._get_resource_tracker()
with rt.resize_claim(context, instance, instance_type, node,
image_meta=image, limits=limits) as claim:
migration, image_meta=image,
limits=limits) as claim:
LOG.info('Migrating', instance=instance)
# RPC cast to the source host to start the actual resize/migration.
self.compute_rpcapi.resize_instance(
@ -3858,7 +3864,7 @@ class ComputeManager(manager.Manager):
@wrap_instance_fault
def prep_resize(self, context, image, instance, instance_type,
reservations, request_spec, filter_properties, node,
clean_shutdown):
clean_shutdown, migration=None):
"""Initiates the process of moving a running instance to another host.
Possibly changes the VCPU, RAM and disk size in the process.
@ -3881,7 +3887,8 @@ class ComputeManager(manager.Manager):
if not isinstance(instance_type, objects.Flavor):
instance_type = objects.Flavor.get_by_id(context,
instance_type['id'])
with self._error_out_instance_on_exception(context, instance):
with self._error_out_instance_on_exception(context, instance), \
errors_out_migration_ctxt(migration):
compute_utils.notify_usage_exists(self.notifier, context, instance,
current_period=True)
self._notify_about_instance_usage(
@ -3890,7 +3897,7 @@ class ComputeManager(manager.Manager):
try:
self._prep_resize(context, image, instance,
instance_type, filter_properties,
node, clean_shutdown)
node, migration, clean_shutdown)
except Exception:
failed = True
# try to re-schedule the resize elsewhere:

14
nova/compute/resource_tracker.py
View File

@ -242,19 +242,19 @@ class ResourceTracker(object):
"""Create a claim for a rebuild operation."""
instance_type = instance.flavor
return self._move_claim(context, instance, instance_type, nodename,
move_type='evacuation', limits=limits,
image_meta=image_meta, migration=migration)
migration, move_type='evacuation',
limits=limits, image_meta=image_meta)
@utils.synchronized(COMPUTE_RESOURCE_SEMAPHORE)
def resize_claim(self, context, instance, instance_type, nodename,
image_meta=None, limits=None):
migration, image_meta=None, limits=None):
"""Create a claim for a resize or cold-migration move."""
return self._move_claim(context, instance, instance_type, nodename,
image_meta=image_meta, limits=limits)
migration, image_meta=image_meta,
limits=limits)
def _move_claim(self, context, instance, new_instance_type, nodename,
move_type=None, image_meta=None, limits=None,
migration=None):
migration, move_type=None, image_meta=None, limits=None):
"""Indicate that resources are needed for a move to this host.
Move can be either a migrate/resize, live-migrate or an
@ -270,7 +270,7 @@ class ResourceTracker(object):
:param limits: Dict of oversubscription limits for memory, disk,
and CPUs
:param migration: A migration object if one was already created
elsewhere for this operation
elsewhere for this operation (otherwise None)
:returns: A Claim ticket representing the reserved resources. This
should be turned into finalize a resource claim or free
resources after the compute operation is finished.

9
nova/compute/rpcapi.py
View File

@ -328,6 +328,7 @@ class ComputeAPI(object):
* 4.15 - Add tag argument to reserve_block_device_name()
* 4.16 - Add tag argument to attach_interface()
* 4.17 - Add new_attachment_id to swap_volume.
* 4.18 - Add migration to prep_resize()
'''
VERSION_ALIASES = {
@ -755,7 +756,7 @@ class ComputeAPI(object):
# TODO(melwitt): Remove the reservations parameter in version 5.0 of the
# RPC API.
def prep_resize(self, ctxt, instance, image, instance_type, host,
reservations=None, request_spec=None,
migration, reservations=None, request_spec=None,
filter_properties=None, node=None,
clean_shutdown=True):
image_p = jsonutils.to_primitive(image)
@ -766,9 +767,13 @@ class ComputeAPI(object):
'request_spec': request_spec,
'filter_properties': filter_properties,
'node': node,
'migration': migration,
'clean_shutdown': clean_shutdown}
version = '4.1'
client = self.router.client(ctxt)
version = '4.18'
if not client.can_send_version(version):
version = '4.1'
del msg_args['migration']
if not client.can_send_version(version):
version = '4.0'
msg_args['instance_type'] = objects_base.obj_to_primitive(

36
nova/conductor/tasks/migrate.py
View File

@ -34,6 +34,34 @@ class MigrationTask(base.TaskBase):
self.compute_rpcapi = compute_rpcapi
self.scheduler_client = scheduler_client
# Persist things from the happy path so we don't have to look
# them up if we need to roll back
self._migration = None
def _preallocate_migration(self):
minver = objects.Service.get_minimum_version_multi(self.context,
['nova-compute'])
if minver < 23:
# NOTE(danms): We can't pre-create the migration since we
# have old computes. Let the compute do it (legacy
# behavior).
return None
migration = objects.Migration(context=self.context.elevated())
migration.old_instance_type_id = self.instance.flavor.id
migration.new_instance_type_id = self.flavor.id
migration.status = 'pre-migrating'
migration.instance_uuid = self.instance.uuid
migration.source_compute = self.instance.host
migration.source_node = self.instance.node
migration.migration_type = (self.instance.flavor.id != self.flavor.id
and 'resize' or 'migration')
migration.create()
self._migration = migration
return migration
def _execute(self):
# TODO(sbauza): Remove that once prep_resize() accepts a RequestSpec
# object in the signature and all the scheduler.utils methods too
@ -64,6 +92,8 @@ class MigrationTask(base.TaskBase):
self.request_spec.requested_destination = objects.Destination(
cell=instance_mapping.cell_mapping)
migration = self._preallocate_migration()
hosts = self.scheduler_client.select_destinations(
self.context, self.request_spec, [self.instance.uuid])
host_state = hosts[0]
@ -88,9 +118,11 @@ class MigrationTask(base.TaskBase):
# RPC cast to the destination host to start the migration process.
self.compute_rpcapi.prep_resize(
self.context, self.instance, legacy_spec['image'],
self.flavor, host, self.reservations,
self.flavor, host, migration, self.reservations,
request_spec=legacy_spec, filter_properties=legacy_props,
node=node, clean_shutdown=self.clean_shutdown)
def rollback(self):
pass
if self._migration:
self._migration.status = 'error'
self._migration.save()

5
nova/objects/service.py
View File

@ -31,7 +31,7 @@ LOG = logging.getLogger(__name__)
# NOTE(danms): This is the global service version counter
SERVICE_VERSION = 22
SERVICE_VERSION = 23
# NOTE(danms): This is our SERVICE_VERSION history. The idea is that any
@ -112,6 +112,9 @@ SERVICE_VERSION_HISTORY = (
# Version 22: A marker for the behaviour change of auto-healing code on the
# compute host regarding allocations against an instance
{'compute_rpc': '4.17'},
# Version 23: Compute hosts allow pre-creation of the migration object
# for cold migration.
{'compute_rpc': '4.18'},
)

52
nova/tests/unit/compute/test_compute_mgr.py
View File

@ -6382,6 +6382,58 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase):
self.assertFalse(do_cleanup)
self.assertFalse(destroy_disks)
@mock.patch('nova.objects.InstanceFault.create')
@mock.patch('nova.objects.Instance.save')
@mock.patch('nova.compute.utils.notify_usage_exists')
@mock.patch('nova.compute.utils.notify_about_instance_usage')
@mock.patch('nova.compute.utils.is_volume_backed_instance',
new=lambda *a: False)
def test_prep_resize_errors_migration(self, mock_niu, mock_notify,
mock_save,
mock_if):
migration = mock.MagicMock()
flavor = objects.Flavor(name='flavor', id=1)
@mock.patch.object(self.compute, '_reschedule')
@mock.patch.object(self.compute, '_prep_resize')
@mock.patch.object(self.compute, '_get_resource_tracker')
def doit(mock_grt, mock_pr, mock_r):
mock_r.return_value = False
mock_pr.side_effect = test.TestingException
instance = objects.Instance(uuid=uuids.instance,
host='host',
node='node',
vm_state='active',
task_state=None)
self.assertRaises(test.TestingException,
self.compute.prep_resize,
self.context, mock.sentinel.image,
instance, flavor,
mock.sentinel.reservations,
mock.sentinel.request_spec,
{}, 'node', False,
migration=migration)
# Make sure we set migration status to failed
self.assertEqual(migration.status, 'error')
# Run it again with migration=None and make sure we don't choke
self.assertRaises(test.TestingException,
self.compute.prep_resize,
self.context, mock.sentinel.image,
instance, flavor,
mock.sentinel.reservations,
mock.sentinel.request_spec,
{}, 'node', False,
migration=None)
# Make sure we only called save once (kinda obviously must be true)
migration.save.assert_called_once_with()
doit()
class ComputeManagerInstanceUsageAuditTestCase(test.TestCase):
def setUp(self):

11
nova/tests/unit/compute/test_resource_tracker.py
View File

@ -1780,7 +1780,8 @@ class TestResize(BaseTestCase):
return_value=mig_context_obj),
mock.patch('nova.objects.Instance.save'),
) as (create_mig_mock, ctxt_mock, inst_save_mock):
claim = self.rt.resize_claim(ctx, instance, new_flavor, _NODENAME)
claim = self.rt.resize_claim(ctx, instance, new_flavor, _NODENAME,
None)
create_mig_mock.assert_called_once_with(
ctx, instance, new_flavor, _NODENAME,
@ -1888,7 +1889,7 @@ class TestResize(BaseTestCase):
return_value=mig_context_obj),
mock.patch('nova.objects.Instance.save'),
) as (create_mig_mock, ctxt_mock, inst_save_mock):
self.rt.resize_claim(ctx, instance, new_flavor, _NODENAME)
self.rt.resize_claim(ctx, instance, new_flavor, _NODENAME, None)
expected = compute_update_usage(expected, new_flavor, sign=1)
self.assertTrue(obj_base.obj_equal_prims(
@ -2014,7 +2015,7 @@ class TestResize(BaseTestCase):
return_value=mig_context_obj),
mock.patch('nova.objects.Instance.save'),
) as (alloc_mock, create_mig_mock, ctxt_mock, inst_save_mock):
self.rt.resize_claim(ctx, instance, new_flavor, _NODENAME)
self.rt.resize_claim(ctx, instance, new_flavor, _NODENAME, None)
pci_claim_mock.assert_called_once_with(ctx, pci_req_mock.return_value,
None)
@ -2177,8 +2178,8 @@ class TestResize(BaseTestCase):
side_effect=[mig_context_obj1, mig_context_obj2]),
mock.patch('nova.objects.Instance.save'),
) as (create_mig_mock, ctxt_mock, inst_save_mock):
self.rt.resize_claim(ctx, instance1, flavor1, _NODENAME)
self.rt.resize_claim(ctx, instance2, flavor2, _NODENAME)
self.rt.resize_claim(ctx, instance1, flavor1, _NODENAME, None)
self.rt.resize_claim(ctx, instance2, flavor2, _NODENAME, None)
cn = self.rt.compute_nodes[_NODENAME]
self.assertTrue(obj_base.obj_equal_prims(expected, cn))
self.assertEqual(2, len(self.rt.tracked_migrations),

9
nova/tests/unit/compute/test_rpcapi.py
View File

@ -137,6 +137,8 @@ class ComputeRpcAPITestCase(test.NoDBTestCase):
kwargs['cast'] = False
else:
kwargs['do_cast'] = False
elif method == 'prep_resize' and 'migration' not in expected_args:
del expected_kwargs['migration']
if 'host' in kwargs:
host = kwargs['host']
elif 'instances' in kwargs:
@ -482,14 +484,16 @@ class ComputeRpcAPITestCase(test.NoDBTestCase):
migrate_data=None, version='4.8')
def test_prep_resize(self):
self._test_compute_api('prep_resize', 'cast',
expected_args = {'migration': 'migration'}
self._test_compute_api('prep_resize', 'cast', expected_args,
instance=self.fake_instance_obj,
instance_type=self.fake_flavor_obj,
image='fake_image', host='host',
reservations=list('fake_res'),
request_spec='fake_spec',
filter_properties={'fakeprop': 'fakeval'},
node='node', clean_shutdown=True, version='4.1')
migration='migration',
node='node', clean_shutdown=True, version='4.18')
self.flags(compute='4.0', group='upgrade_levels')
expected_args = {'instance_type': self.fake_flavor}
self._test_compute_api('prep_resize', 'cast', expected_args,
@ -499,6 +503,7 @@ class ComputeRpcAPITestCase(test.NoDBTestCase):
reservations=list('fake_res'),
request_spec='fake_spec',
filter_properties={'fakeprop': 'fakeval'},
migration='migration',
node='node', clean_shutdown=True, version='4.0')
def test_reboot_instance(self):

79
nova/tests/unit/conductor/tasks/test_migrate.py
View File

@ -54,15 +54,19 @@ class MigrationTaskTestCase(test.NoDBTestCase):
compute_rpcapi.ComputeAPI(),
scheduler_client.SchedulerClient())
@mock.patch('nova.objects.Service.get_minimum_version_multi')
@mock.patch('nova.availability_zones.get_host_availability_zone')
@mock.patch.object(scheduler_utils, 'setup_instance_group')
@mock.patch.object(scheduler_client.SchedulerClient, 'select_destinations')
@mock.patch.object(compute_rpcapi.ComputeAPI, 'prep_resize')
def test_execute(self, prep_resize_mock, sel_dest_mock, sig_mock, az_mock):
def test_execute_legacy_no_pre_create_migration(self, prep_resize_mock,
sel_dest_mock, sig_mock,
az_mock, gmv_mock):
sel_dest_mock.return_value = self.hosts
az_mock.return_value = 'myaz'
task = self._generate_task()
legacy_request_spec = self.request_spec.to_legacy_request_spec_dict()
gmv_mock.return_value = 22
task.execute()
sig_mock.assert_called_once_with(self.context, self.request_spec)
@ -70,8 +74,79 @@ class MigrationTaskTestCase(test.NoDBTestCase):
self.context, self.request_spec, [self.instance.uuid])
prep_resize_mock.assert_called_once_with(
self.context, self.instance, legacy_request_spec['image'],
self.flavor, self.hosts[0]['host'], self.reservations,
self.flavor, self.hosts[0]['host'], None, self.reservations,
request_spec=legacy_request_spec,
filter_properties=self.filter_properties,
node=self.hosts[0]['nodename'], clean_shutdown=self.clean_shutdown)
az_mock.assert_called_once_with(self.context, 'host1')
self.assertIsNone(task._migration)
@mock.patch('nova.objects.Migration.create')
@mock.patch('nova.objects.Service.get_minimum_version_multi')
@mock.patch('nova.availability_zones.get_host_availability_zone')
@mock.patch.object(scheduler_utils, 'setup_instance_group')
@mock.patch.object(scheduler_client.SchedulerClient, 'select_destinations')
@mock.patch.object(compute_rpcapi.ComputeAPI, 'prep_resize')
def test_execute(self, prep_resize_mock, sel_dest_mock, sig_mock, az_mock,
gmv_mock, cm_mock):
sel_dest_mock.return_value = self.hosts
az_mock.return_value = 'myaz'
task = self._generate_task()
legacy_request_spec = self.request_spec.to_legacy_request_spec_dict()
gmv_mock.return_value = 23
task.execute()
sig_mock.assert_called_once_with(self.context, self.request_spec)
task.scheduler_client.select_destinations.assert_called_once_with(
self.context, self.request_spec, [self.instance.uuid])
prep_resize_mock.assert_called_once_with(
self.context, self.instance, legacy_request_spec['image'],
self.flavor, self.hosts[0]['host'], task._migration,
self.reservations, request_spec=legacy_request_spec,
filter_properties=self.filter_properties,
node=self.hosts[0]['nodename'], clean_shutdown=self.clean_shutdown)
az_mock.assert_called_once_with(self.context, 'host1')
self.assertIsNotNone(task._migration)
old_flavor = self.instance.flavor
new_flavor = self.flavor
self.assertEqual(old_flavor.id, task._migration.old_instance_type_id)
self.assertEqual(new_flavor.id, task._migration.new_instance_type_id)
self.assertEqual('pre-migrating', task._migration.status)
self.assertEqual(self.instance.uuid, task._migration.instance_uuid)
self.assertEqual(self.instance.host, task._migration.source_compute)
self.assertEqual(self.instance.node, task._migration.source_node)
if old_flavor.id != new_flavor.id:
self.assertEqual('resize', task._migration.migration_type)
else:
self.assertEqual('migration', task._migration.migration_type)
task._migration.create.assert_called_once_with()
def test_execute_resize(self):
self.flavor = self.flavor.obj_clone()
self.flavor.id = 3
self.test_execute()
@mock.patch('nova.scheduler.client.report.SchedulerReportClient')
@mock.patch('nova.objects.ComputeNode.get_by_host_and_nodename')
@mock.patch('nova.objects.Migration.save')
@mock.patch('nova.objects.Migration.create')
@mock.patch('nova.objects.Service.get_minimum_version_multi')
@mock.patch('nova.availability_zones.get_host_availability_zone')
@mock.patch.object(scheduler_utils, 'setup_instance_group')
@mock.patch.object(scheduler_client.SchedulerClient, 'select_destinations')
@mock.patch.object(compute_rpcapi.ComputeAPI, 'prep_resize')
def test_execute_rollback(self, prep_resize_mock, sel_dest_mock, sig_mock,
az_mock, gmv_mock, cm_mock, sm_mock, cn_mock,
rc_mock):
sel_dest_mock.return_value = self.hosts
az_mock.return_value = 'myaz'
task = self._generate_task()
gmv_mock.return_value = 23
prep_resize_mock.side_effect = test.TestingException
self.assertRaises(test.TestingException, task.execute)
self.assertIsNotNone(task._migration)
task._migration.create.assert_called_once_with()
task._migration.save.assert_called_once_with()
self.assertEqual('error', task._migration.status)

2
nova/tests/unit/conductor/test_conductor.py
View File

@ -2305,7 +2305,7 @@ class ConductorTaskTestCase(_BaseTaskTestCase, test_compute.BaseTestCase):
self.context, fake_spec, [inst_obj.uuid])
prep_resize_mock.assert_called_once_with(
self.context, inst_obj, legacy_request_spec['image'],
flavor, hosts[0]['host'], [resvs],
flavor, hosts[0]['host'], None, [resvs],
request_spec=legacy_request_spec,
filter_properties=legacy_filter_props,
node=hosts[0]['nodename'], clean_shutdown=True)