From fa002925460e70d988d1b4dd1ea594c680a43740 Mon Sep 17 00:00:00 2001 From: Andrea Rosa Date: Fri, 5 Feb 2016 08:31:06 +0000 Subject: [PATCH] Abort an ongoing live migration This change adds a DELETE call on the server-migrations object to cancel a running live migration of a specific instance. TO perform the cancellation the virtualization driver needs to support it, in case that the feature is not supported we return an error. We allow a cancellation of a migration only if the migration is running at the moment of the request and if the migration type is equal to 'live-migration'. In this change we implement this feature for the libvirt driver. When the cancellation of a live migration succeeded we rollback the live migration and we set the state of the Migration object equals to 'cancelled'. The implementation of this change is based on the work done by the implementation of the feature called 'force live migration': https://review.openstack.org/245921 DocImpact ApiImpact Implements blueprint: abort-live-migration Change-Id: I1ff861e54997a069894b542bd764ac3ef1b3dbb2 --- .../versions/v21-version-get-resp.json | 2 +- .../versions/versions-get-resp.json | 2 +- doc/notification_samples/service-update.json | 2 +- etc/nova/policy.json | 1 + nova/api/openstack/api_version_request.py | 4 +- .../openstack/compute/server_migrations.py | 19 +++++ .../openstack/rest_api_version_history.rst | 7 ++ nova/compute/api.py | 27 +++++++ nova/compute/instance_actions.py | 1 + nova/compute/manager.py | 34 +++++++- nova/compute/rpcapi.py | 8 ++ nova/objects/service.py | 4 +- .../test_server_migrations.py | 51 ++++++++++++ .../compute/test_server_migrations.py | 77 +++++++++++++++++++ nova/tests/unit/compute/test_compute.py | 24 ++++++ nova/tests/unit/compute/test_compute_api.py | 41 ++++++++++ nova/tests/unit/compute/test_compute_mgr.py | 62 +++++++++++++++ nova/tests/unit/compute/test_rpcapi.py | 5 ++ nova/tests/unit/fake_policy.py | 1 + nova/tests/unit/test_policy.py | 1 + nova/tests/unit/virt/libvirt/test_driver.py | 31 ++++++-- nova/tests/unit/virt/test_virt_drivers.py | 5 ++ nova/virt/driver.py | 8 ++ nova/virt/fake.py | 3 + nova/virt/libvirt/driver.py | 19 ++++- ...abort-live-migration-cb902bb0754b11b6.yaml | 5 ++ 26 files changed, 429 insertions(+), 15 deletions(-) create mode 100644 releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml diff --git a/doc/api_samples/versions/v21-version-get-resp.json b/doc/api_samples/versions/v21-version-get-resp.json index efbbf344babb..afb4e3dcc294 100644 --- a/doc/api_samples/versions/v21-version-get-resp.json +++ b/doc/api_samples/versions/v21-version-get-resp.json @@ -19,7 +19,7 @@ } ], "status": "CURRENT", - "version": "2.23", + "version": "2.24", "min_version": "2.1", "updated": "2013-07-23T11:33:21Z" } diff --git a/doc/api_samples/versions/versions-get-resp.json b/doc/api_samples/versions/versions-get-resp.json index 6b4eaada1ac2..1e34a553e7ec 100644 --- a/doc/api_samples/versions/versions-get-resp.json +++ b/doc/api_samples/versions/versions-get-resp.json @@ -22,7 +22,7 @@ } ], "status": "CURRENT", - "version": "2.23", + "version": "2.24", "min_version": "2.1", "updated": "2013-07-23T11:33:21Z" } diff --git a/doc/notification_samples/service-update.json b/doc/notification_samples/service-update.json index 219dec9ae2ea..f1e7e0bd9e7b 100644 --- a/doc/notification_samples/service-update.json +++ b/doc/notification_samples/service-update.json @@ -13,7 +13,7 @@ "disabled_reason": null, "report_count": 1, "forced_down": false, - "version": 7 + "version": 8 } }, "event_type": "service.update", diff --git a/etc/nova/policy.json b/etc/nova/policy.json index 3877c99edbef..dc35f0c20444 100644 --- a/etc/nova/policy.json +++ b/etc/nova/policy.json @@ -271,6 +271,7 @@ "os_compute_api:servers:stop": "rule:admin_or_owner", "os_compute_api:servers:trigger_crash_dump": "rule:admin_or_owner", "os_compute_api:servers:migrations:force_complete": "rule:admin_api", + "os_compute_api:servers:migrations:delete": "rule:admin_api", "os_compute_api:servers:discoverable": "@", "os_compute_api:servers:migrations:index": "rule:admin_api", "os_compute_api:servers:migrations:show": "rule:admin_api", diff --git a/nova/api/openstack/api_version_request.py b/nova/api/openstack/api_version_request.py index 77a358cc4bec..33f364d99900 100644 --- a/nova/api/openstack/api_version_request.py +++ b/nova/api/openstack/api_version_request.py @@ -68,6 +68,8 @@ REST_API_VERSION_HISTORY = """REST API Version History: * 2.23 - Add index/show API for server migrations. Also add migration_type for /os-migrations and add ref link for it when the migration is an in progress live migration. + * 2.24 - Add API to cancel a running live migration + """ # The minimum and maximum versions of the API supported @@ -76,7 +78,7 @@ REST_API_VERSION_HISTORY = """REST API Version History: # Note(cyeoh): This only applies for the v2.1 API once microversions # support is fully merged. It does not affect the V2 API. _MIN_API_VERSION = "2.1" -_MAX_API_VERSION = "2.23" +_MAX_API_VERSION = "2.24" DEFAULT_API_VERSION = _MIN_API_VERSION diff --git a/nova/api/openstack/compute/server_migrations.py b/nova/api/openstack/compute/server_migrations.py index 7acf6c4e4f08..fa5ba5e5b579 100644 --- a/nova/api/openstack/compute/server_migrations.py +++ b/nova/api/openstack/compute/server_migrations.py @@ -135,6 +135,25 @@ class ServerMigrationsController(wsgi.Controller): return {'migration': output(migration)} + @wsgi.Controller.api_version("2.24") + @wsgi.response(202) + @extensions.expected_errors((400, 404, 409)) + def delete(self, req, server_id, id): + """Abort an in progress migration of an instance.""" + context = req.environ['nova.context'] + authorize(context, action="delete") + + instance = common.get_instance(self.compute_api, context, server_id) + try: + self.compute_api.live_migrate_abort(context, instance, id) + except exception.InstanceInvalidState as state_error: + common.raise_http_conflict_for_instance_invalid_state( + state_error, "abort live migration", server_id) + except exception.MigrationNotFoundForInstance as e: + raise exc.HTTPNotFound(explanation=e.format_message()) + except exception.InvalidMigrationState as e: + raise exc.HTTPBadRequest(explanation=e.format_message()) + class ServerMigrations(extensions.V21APIExtensionBase): """Server Migrations API.""" diff --git a/nova/api/openstack/rest_api_version_history.rst b/nova/api/openstack/rest_api_version_history.rst index 85c45087becf..c363136c52dc 100644 --- a/nova/api/openstack/rest_api_version_history.rst +++ b/nova/api/openstack/rest_api_version_history.rst @@ -205,3 +205,10 @@ user documentation. Add migration_type for old /os-migrations API, also add ref link to the /servers/{uuid}/migrations/{id} for it when the migration is an in-progress live-migration. + +2.24 +--- + + A new API call to cancel a running live migration:: + + DELETE /servers//migrations/ diff --git a/nova/compute/api.py b/nova/compute/api.py index c4e0ee221817..a7b0d5e9f687 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -3328,6 +3328,33 @@ class API(base.Base): self.compute_rpcapi.live_migration_force_complete( context, instance, migration.id) + @check_instance_lock + @check_instance_cell + @check_instance_state(task_state=[task_states.MIGRATING]) + def live_migrate_abort(self, context, instance, migration_id): + """Abort an in-progress live migration. + + :param context: Security context + :param instance: The instance that is being migrated + :param migration_id: ID of in-progress live migration + + """ + migration = objects.Migration.get_by_id_and_instance(context, + migration_id, instance.uuid) + LOG.debug("Going to cancel live migration %s", + migration.id, instance=instance) + + if migration.status != 'running': + raise exception.InvalidMigrationState(migration_id=migration_id, + instance_uuid=instance.uuid, + state=migration.status, + method='abort live migration') + self._record_action_start(context, instance, + instance_actions.LIVE_MIGRATION_CANCEL) + + self.compute_rpcapi.live_migration_abort(context, + instance, migration.id) + @check_instance_state(vm_state=[vm_states.ACTIVE, vm_states.STOPPED, vm_states.ERROR]) def evacuate(self, context, instance, host, on_shared_storage, diff --git a/nova/compute/instance_actions.py b/nova/compute/instance_actions.py index 1fbe7b1429ba..a08e660b98ed 100644 --- a/nova/compute/instance_actions.py +++ b/nova/compute/instance_actions.py @@ -49,4 +49,5 @@ CHANGE_PASSWORD = 'changePassword' SHELVE = 'shelve' UNSHELVE = 'unshelve' LIVE_MIGRATION = 'live-migration' +LIVE_MIGRATION_CANCEL = 'live_migration_cancel' TRIGGER_CRASH_DUMP = 'trigger_crash_dump' diff --git a/nova/compute/manager.py b/nova/compute/manager.py index a3d0669628f0..49d0317690df 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -674,7 +674,7 @@ class ComputeVirtAPI(virtapi.VirtAPI): class ComputeManager(manager.Manager): """Manages the running instances from creation to destruction.""" - target = messaging.Target(version='4.9') + target = messaging.Target(version='4.10') # How long to wait in seconds before re-issuing a shutdown # signal to an instance during power off. The overall @@ -5278,6 +5278,30 @@ class ComputeManager(manager.Manager): self._notify_about_instance_usage( context, instance, 'live.migration.force.complete.end') + @wrap_exception() + @wrap_instance_event + @wrap_instance_fault + def live_migration_abort(self, context, instance, migration_id): + """Abort an in-progress live migration. + + :param context: Security context + :param instance: The instance that is being migrated + :param migration_id: ID of in-progress live migration + + """ + migration = objects.Migration.get_by_id(context, migration_id) + if migration.status != 'running': + raise exception.InvalidMigrationState(migration_id=migration_id, + instance_uuid=instance.uuid, + state=migration.status, + method='abort live migration') + + self._notify_about_instance_usage( + context, instance, 'live.migration.abort.start') + self.driver.live_migration_abort(instance) + self._notify_about_instance_usage( + context, instance, 'live.migration.abort.end') + def _live_migration_cleanup_flags(self, block_migration, migrate_data): """Determine whether disks or instance path need to be cleaned up after live migration (at source on success, at destination on rollback) @@ -5509,7 +5533,8 @@ class ComputeManager(manager.Manager): @wrap_exception() @wrap_instance_fault def _rollback_live_migration(self, context, instance, - dest, block_migration, migrate_data=None): + dest, block_migration, migrate_data=None, + migration_status='error'): """Recovers Instance/volume state from migrating -> running. :param context: security context @@ -5520,6 +5545,8 @@ class ComputeManager(manager.Manager): :param block_migration: if true, prepare for block migration :param migrate_data: if not none, contains implementation specific data. + :param migration_status: + Contains the status we want to set for the migration object """ instance.task_state = None @@ -5559,7 +5586,8 @@ class ComputeManager(manager.Manager): self._notify_about_instance_usage(context, instance, "live_migration._rollback.end") - self._set_migration_status(migration, 'error') + + self._set_migration_status(migration, migration_status) @wrap_exception() @wrap_instance_event diff --git a/nova/compute/rpcapi.py b/nova/compute/rpcapi.py index c9b33c6b3a52..f78a307b5241 100644 --- a/nova/compute/rpcapi.py +++ b/nova/compute/rpcapi.py @@ -326,6 +326,7 @@ class ComputeAPI(object): pre_live_migration. * ... - Remove refresh_provider_fw_rules() * 4.9 - Add live_migration_force_complete() + * 4.10 - Add live_migration_abort() ''' VERSION_ALIASES = { @@ -644,6 +645,13 @@ class ComputeAPI(object): cctxt.cast(ctxt, 'live_migration_force_complete', instance=instance, migration_id=migration_id) + def live_migration_abort(self, ctxt, instance, migration_id): + version = '4.10' + cctxt = self.client.prepare(server=_compute_host(None, instance), + version=version) + cctxt.cast(ctxt, 'live_migration_abort', instance=instance, + migration_id=migration_id) + def pause_instance(self, ctxt, instance): version = '4.0' cctxt = self.client.prepare(server=_compute_host(None, instance), diff --git a/nova/objects/service.py b/nova/objects/service.py index c7a5ee3ab907..15062a8d447e 100644 --- a/nova/objects/service.py +++ b/nova/objects/service.py @@ -29,7 +29,7 @@ LOG = logging.getLogger(__name__) # NOTE(danms): This is the global service version counter -SERVICE_VERSION = 7 +SERVICE_VERSION = 8 # NOTE(danms): This is our SERVICE_VERSION history. The idea is that any @@ -67,6 +67,8 @@ SERVICE_VERSION_HISTORY = ( {'compute_rpc': '4.8'}, # Version 7: Add live_migration_force_complete in the compute_rpc {'compute_rpc': '4.9'}, + # Version 8: Add live_migration_abort in the compute_rpc + {'compute_rpc': '4.10'}, ) diff --git a/nova/tests/functional/api_sample_tests/test_server_migrations.py b/nova/tests/functional/api_sample_tests/test_server_migrations.py index 88d5279d72fd..a49743bde02f 100644 --- a/nova/tests/functional/api_sample_tests/test_server_migrations.py +++ b/nova/tests/functional/api_sample_tests/test_server_migrations.py @@ -158,3 +158,54 @@ class ServerMigrationsSamplesJsonTestV2_23(test_servers.ServersSampleBase): self._verify_response('migrations-index', {"server_uuid_1": self.UUID_1}, response, 200) + + +class ServerMigrationsSampleJsonTestV2_24(test_servers.ServersSampleBase): + ADMIN_API = True + extension_name = "server-migrations" + scenarios = [('v2_24', {'api_major_version': 'v2.1'})] + extra_extensions_to_load = ["os-migrate-server", "os-access-ips"] + + def setUp(self): + """setUp method for server usage.""" + super(ServerMigrationsSampleJsonTestV2_24, self).setUp() + self.uuid = self._post_server() + self.context = context.RequestContext('fake', 'fake') + fake_migration = { + 'source_node': self.compute.host, + 'dest_node': 'node10', + 'source_compute': 'compute1', + 'dest_compute': 'compute12', + 'migration_type': 'live-migration', + 'instance_uuid': self.uuid, + 'status': 'running'} + + self.migration = objects.Migration(context=self.context, + **fake_migration) + self.migration.create() + + @mock.patch.object(conductor_manager.ComputeTaskManager, '_live_migrate') + def test_live_migrate_abort(self, _live_migrate): + self._do_post('servers/%s/action' % self.uuid, 'live-migrate-server', + {'hostname': self.compute.host}) + uri = 'servers/%s/migrations/%s' % (self.uuid, self.migration.id) + response = self._do_delete(uri, api_version='2.24') + self.assertEqual(202, response.status_code) + + @mock.patch.object(conductor_manager.ComputeTaskManager, '_live_migrate') + def test_live_migrate_abort_migration_not_found(self, _live_migrate): + self._do_post('servers/%s/action' % self.uuid, 'live-migrate-server', + {'hostname': self.compute.host}) + uri = 'servers/%s/migrations/%s' % (self.uuid, '45') + response = self._do_delete(uri, api_version='2.24') + self.assertEqual(404, response.status_code) + + @mock.patch.object(conductor_manager.ComputeTaskManager, '_live_migrate') + def test_live_migrate_abort_migration_not_running(self, _live_migrate): + self.migration.status = 'completed' + self.migration.save() + self._do_post('servers/%s/action' % self.uuid, 'live-migrate-server', + {'hostname': self.compute.host}) + uri = 'servers/%s/migrations/%s' % (self.uuid, self.migration.id) + response = self._do_delete(uri, api_version='2.24') + self.assertEqual(400, response.status_code) diff --git a/nova/tests/unit/api/openstack/compute/test_server_migrations.py b/nova/tests/unit/api/openstack/compute/test_server_migrations.py index a63b1d5dc224..6a18d6a72e57 100644 --- a/nova/tests/unit/api/openstack/compute/test_server_migrations.py +++ b/nova/tests/unit/api/openstack/compute/test_server_migrations.py @@ -261,6 +261,67 @@ class ServerMigrationsTestsV223(ServerMigrationsTestsV21): want_objects=True) +class ServerMigrationsTestsV224(ServerMigrationsTestsV21): + wsgi_api_version = '2.24' + + def setUp(self): + super(ServerMigrationsTestsV224, self).setUp() + self.req = fakes.HTTPRequest.blank('', version=self.wsgi_api_version, + use_admin_context=True) + self.context = self.req.environ['nova.context'] + + def test_cancel_live_migration_succeeded(self): + @mock.patch.object(self.compute_api, 'live_migrate_abort') + @mock.patch.object(self.compute_api, 'get') + def _do_test(mock_get, mock_abort): + self.controller.delete(self.req, 'server-id', 'migration-id') + mock_abort.assert_called_once_with(self.context, + mock_get(), + 'migration-id') + _do_test() + + def _test_cancel_live_migration_failed(self, fake_exc, expected_exc): + @mock.patch.object(self.compute_api, 'live_migrate_abort', + side_effect=fake_exc) + @mock.patch.object(self.compute_api, 'get') + def _do_test(mock_get, mock_abort): + self.assertRaises(expected_exc, + self.controller.delete, + self.req, + 'server-id', + 'migration-id') + _do_test() + + def test_cancel_live_migration_invalid_state(self): + self._test_cancel_live_migration_failed( + exception.InstanceInvalidState(instance_uuid='', + state='', + attr='', + method=''), + webob.exc.HTTPConflict) + + def test_cancel_live_migration_migration_not_found(self): + self._test_cancel_live_migration_failed( + exception.MigrationNotFoundForInstance(migration_id='', + instance_id=''), + webob.exc.HTTPNotFound) + + def test_cancel_live_migration_invalid_migration_state(self): + self._test_cancel_live_migration_failed( + exception.InvalidMigrationState(migration_id='', + instance_uuid='', + state='', + method=''), + webob.exc.HTTPBadRequest) + + def test_cancel_live_migration_instance_not_found(self): + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.delete, + self.req, + 'server-id', + 'migration-id') + + class ServerMigrationsPolicyEnforcementV21(test.NoDBTestCase): wsgi_api_version = '2.22' @@ -308,3 +369,19 @@ class ServerMigrationsPolicyEnforcementV223( fakes.FAKE_UUID, 1) self.assertEqual("Policy doesn't allow %s to be performed." % rule_name, exc.format_message()) + + +class ServerMigrationsPolicyEnforcementV224( + ServerMigrationsPolicyEnforcementV223): + + wsgi_api_version = '2.24' + + def setUp(self): + super(ServerMigrationsPolicyEnforcementV224, self).setUp() + + def test_migrate_delete_failed(self): + rule_name = "os_compute_api:servers:migrations:delete" + self.policy.set_rules({rule_name: "project:non_fake"}) + self.assertRaises(exception.PolicyNotAuthorized, + self.controller.delete, self.req, + fakes.FAKE_UUID, '10') diff --git a/nova/tests/unit/compute/test_compute.py b/nova/tests/unit/compute/test_compute.py index 0311b19f220c..f64476421869 100644 --- a/nova/tests/unit/compute/test_compute.py +++ b/nova/tests/unit/compute/test_compute.py @@ -5849,6 +5849,30 @@ class ComputeTestCase(BaseTestCase): self.assertEqual('error', migration.status) migration.save.assert_called_once_with() + @mock.patch('nova.objects.BlockDeviceMappingList.get_by_instance_uuid') + def test_rollback_live_migration_set_migration_status(self, mock_bdms): + c = context.get_admin_context() + instance = mock.MagicMock() + migration = mock.MagicMock() + migrate_data = {'migration': migration} + + mock_bdms.return_value = [] + + @mock.patch.object(self.compute, '_live_migration_cleanup_flags') + @mock.patch.object(self.compute, 'network_api') + def _test(mock_nw_api, mock_lmcf): + mock_lmcf.return_value = False, False + self.compute._rollback_live_migration(c, instance, 'foo', + False, + migrate_data=migrate_data, + migration_status='fake') + mock_nw_api.setup_networks_on_host.assert_called_once_with( + c, instance, self.compute.host) + _test() + + self.assertEqual('fake', migration.status) + migration.save.assert_called_once_with() + def test_rollback_live_migration_at_destination_correctly(self): # creating instance testdata c = context.get_admin_context() diff --git a/nova/tests/unit/compute/test_compute_api.py b/nova/tests/unit/compute/test_compute_api.py index 701dcca359d3..e7755284c8a5 100644 --- a/nova/tests/unit/compute/test_compute_api.py +++ b/nova/tests/unit/compute/test_compute_api.py @@ -3266,6 +3266,47 @@ class _ComputeAPIUnitTestMixIn(object): self.compute_api.live_migrate_force_complete, self.context, instance, '1') + def _get_migration(self, migration_id, status, migration_type): + migration = objects.Migration() + migration.id = migration_id + migration.status = status + migration.migration_type = migration_type + return migration + + @mock.patch('nova.compute.api.API._record_action_start') + @mock.patch.object(compute_rpcapi.ComputeAPI, 'live_migration_abort') + @mock.patch.object(objects.Migration, 'get_by_id_and_instance') + def test_live_migrate_abort_succeeded(self, + mock_get_migration, + mock_lm_abort, + mock_rec_action): + instance = self._create_instance_obj() + instance.task_state = task_states.MIGRATING + migration = self._get_migration(21, 'running', 'live-migration') + mock_get_migration.return_value = migration + + self.compute_api.live_migrate_abort(self.context, + instance, + migration.id) + mock_rec_action.assert_called_once_with(self.context, + instance, + instance_actions.LIVE_MIGRATION_CANCEL) + mock_lm_abort.called_once_with(self.context, instance, migration.id) + + @mock.patch.object(objects.Migration, 'get_by_id_and_instance') + def test_live_migration_abort_wrong_migration_status(self, + mock_get_migration): + instance = self._create_instance_obj() + instance.task_state = task_states.MIGRATING + migration = self._get_migration(21, 'completed', 'live-migration') + mock_get_migration.return_value = migration + + self.assertRaises(exception.InvalidMigrationState, + self.compute_api.live_migrate_abort, + self.context, + instance, + migration.id) + class ComputeAPIUnitTestCase(_ComputeAPIUnitTestMixIn, test.NoDBTestCase): def setUp(self): diff --git a/nova/tests/unit/compute/test_compute_mgr.py b/nova/tests/unit/compute/test_compute_mgr.py index 9b484959d20a..17e812418dba 100644 --- a/nova/tests/unit/compute/test_compute_mgr.py +++ b/nova/tests/unit/compute/test_compute_mgr.py @@ -4526,6 +4526,68 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase): _do_test() + def _get_migration(self, migration_id, status, migration_type): + migration = objects.Migration() + migration.id = migration_id + migration.status = status + migration.migration_type = migration_type + return migration + + @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage') + @mock.patch.object(objects.Migration, 'get_by_id') + @mock.patch.object(nova.virt.fake.SmallFakeDriver, 'live_migration_abort') + def test_live_migration_abort(self, + mock_driver, + mock_get_migration, + mock_notify): + instance = objects.Instance(id=123, uuid=uuids.instance) + migration = self._get_migration(10, 'running', 'live-migration') + mock_get_migration.return_value = migration + self.compute.live_migration_abort(self.context, instance, migration.id) + + mock_driver.assert_called_with(instance) + _notify_usage_calls = [mock.call(self.context, + instance, + 'live.migration.abort.start'), + mock.call(self.context, + instance, + 'live.migration.abort.end')] + + mock_notify.assert_has_calls(_notify_usage_calls) + + @mock.patch.object(compute_utils, 'add_instance_fault_from_exc') + @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage') + @mock.patch.object(objects.Migration, 'get_by_id') + @mock.patch.object(nova.virt.fake.SmallFakeDriver, 'live_migration_abort') + def test_live_migration_abort_not_supported(self, + mock_driver, + mock_get_migration, + mock_notify, + mock_instance_fault): + instance = objects.Instance(id=123, uuid=uuids.instance) + migration = self._get_migration(10, 'running', 'live-migration') + mock_get_migration.return_value = migration + mock_driver.side_effect = NotImplementedError() + self.assertRaises(NotImplementedError, + self.compute.live_migration_abort, + self.context, + instance, + migration.id) + + @mock.patch.object(compute_utils, 'add_instance_fault_from_exc') + @mock.patch.object(objects.Migration, 'get_by_id') + def test_live_migration_abort_wrong_migration_state(self, + mock_get_migration, + mock_instance_fault): + instance = objects.Instance(id=123, uuid=uuids.instance) + migration = self._get_migration(10, 'completed', 'live-migration') + mock_get_migration.return_value = migration + self.assertRaises(exception.InvalidMigrationState, + self.compute.live_migration_abort, + self.context, + instance, + migration.id) + class ComputeManagerInstanceUsageAuditTestCase(test.TestCase): def setUp(self): diff --git a/nova/tests/unit/compute/test_rpcapi.py b/nova/tests/unit/compute/test_rpcapi.py index 8396adef924b..54af080d9150 100644 --- a/nova/tests/unit/compute/test_rpcapi.py +++ b/nova/tests/unit/compute/test_rpcapi.py @@ -309,6 +309,11 @@ class ComputeRpcAPITestCase(test.NoDBTestCase): instance=self.fake_instance_obj, migration_id='1', version='4.9') + def test_live_migration_abort(self): + self._test_compute_api('live_migration_abort', 'cast', + instance=self.fake_instance_obj, + migration_id='1', version='4.10') + def test_post_live_migration_at_destination(self): self._test_compute_api('post_live_migration_at_destination', 'cast', instance=self.fake_instance_obj, diff --git a/nova/tests/unit/fake_policy.py b/nova/tests/unit/fake_policy.py index 42cae2140007..540a3b164bdb 100644 --- a/nova/tests/unit/fake_policy.py +++ b/nova/tests/unit/fake_policy.py @@ -125,6 +125,7 @@ policy_data = """ "os_compute_api:servers:start": "", "os_compute_api:servers:stop": "", "os_compute_api:servers:trigger_crash_dump": "", + "os_compute_api:servers:migrations:delete": "rule:admin_api", "os_compute_api:servers:migrations:force_complete": "", "os_compute_api:servers:migrations:index": "rule:admin_api", "os_compute_api:servers:migrations:show": "rule:admin_api", diff --git a/nova/tests/unit/test_policy.py b/nova/tests/unit/test_policy.py index 1d8da7914e58..998106a61f00 100644 --- a/nova/tests/unit/test_policy.py +++ b/nova/tests/unit/test_policy.py @@ -301,6 +301,7 @@ class RealRolePolicyTestCase(test.NoDBTestCase): "os_compute_api:servers:index:get_all_tenants", "os_compute_api:servers:show:host_status", "os_compute_api:servers:migrations:force_complete", +"os_compute_api:servers:migrations:delete", "network:attach_external_network", "os_compute_api:os-admin-actions", "os_compute_api:os-admin-actions:reset_network", diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index 0ab16cac0a77..14364493f863 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -7677,7 +7677,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): mock_mig_save, mock_job_info, mock_sleep, - mock_time): + mock_time, + expected_mig_status=None): drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) instance = objects.Instance(**self.test_instance) dom = fakelibvirt.Domain(drvr._get_connection(), "", True) @@ -7746,8 +7747,13 @@ class LibvirtConnTestCase(test.NoDBTestCase): 'abortJob not called when failure expected') self.assertFalse(fake_post_method.called, 'Post method called when success not expected') - fake_recover_method.assert_called_once_with( - self.context, instance, dest, False, migrate_data) + if expected_mig_status: + fake_recover_method.assert_called_once_with( + self.context, instance, dest, False, migrate_data, + migration_status=expected_mig_status) + else: + fake_recover_method.assert_called_once_with( + self.context, instance, dest, False, migrate_data) def test_live_migration_monitor_success(self): # A normal sequence where see all the normal job states @@ -7847,7 +7853,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): ] self._test_live_migration_monitoring(domain_info_records, [], - self.EXPECT_FAILURE) + self.EXPECT_FAILURE, + expected_mig_status='cancelled') @mock.patch.object(fakelibvirt.virDomain, "migrateSetMaxDowntime") @mock.patch.object(libvirt_driver.LibvirtDriver, @@ -7929,7 +7936,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): ] self._test_live_migration_monitoring(domain_info_records, - fake_times, self.EXPECT_ABORT) + fake_times, self.EXPECT_ABORT, + expected_mig_status='cancelled') def test_live_migration_monitor_progress(self): self.flags(live_migration_completion_timeout=1000000, @@ -7960,7 +7968,8 @@ class LibvirtConnTestCase(test.NoDBTestCase): ] self._test_live_migration_monitoring(domain_info_records, - fake_times, self.EXPECT_ABORT) + fake_times, self.EXPECT_ABORT, + expected_mig_status='cancelled') def test_live_migration_downtime_steps(self): self.flags(live_migration_downtime=400, group='libvirt') @@ -13329,6 +13338,16 @@ class LibvirtConnTestCase(test.NoDBTestCase): drvr.live_migration_force_complete(self.test_instance) pause.assert_called_once_with(self.test_instance) + @mock.patch.object(fakelibvirt.virDomain, "abortJob") + def test_live_migration_abort(self, mock_abort): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + dom = fakelibvirt.Domain(drvr._get_connection(), "", False) + guest = libvirt_guest.Guest(dom) + with mock.patch.object(nova.virt.libvirt.host.Host, 'get_guest', + return_value=guest): + drvr.live_migration_abort(self.test_instance) + self.assertTrue(mock_abort.called) + @mock.patch('os.path.exists', return_value=True) @mock.patch('tempfile.mkstemp') @mock.patch('os.close', return_value=None) diff --git a/nova/tests/unit/virt/test_virt_drivers.py b/nova/tests/unit/virt/test_virt_drivers.py index b592cc422ff1..78500aa81f01 100644 --- a/nova/tests/unit/virt/test_virt_drivers.py +++ b/nova/tests/unit/virt/test_virt_drivers.py @@ -672,6 +672,11 @@ class _VirtDriverTestCase(_FakeDriverBackendTestCase): instance_ref, network_info = self._get_running_instance() self.connection.live_migration_force_complete(instance_ref) + @catch_notimplementederror + def test_live_migration_abort(self): + instance_ref, network_info = self._get_running_instance() + self.connection.live_migration_abort(instance_ref) + @catch_notimplementederror def _check_available_resource_fields(self, host_status): keys = ['vcpus', 'memory_mb', 'local_gb', 'vcpus_used', diff --git a/nova/virt/driver.py b/nova/virt/driver.py index 624b01490741..6d5090e3e8e1 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -837,6 +837,14 @@ class ComputeDriver(object): """ raise NotImplementedError() + def live_migration_abort(self, instance): + """Abort an in-progress live migration. + + :param instance: instance that is live migrating + + """ + raise NotImplementedError() + def rollback_live_migration_at_destination(self, context, instance, network_info, block_device_info, diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 1c0e9cfe42dc..5b305492e7b7 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -471,6 +471,9 @@ class FakeDriver(driver.ComputeDriver): def live_migration_force_complete(self, instance): return + def live_migration_abort(self, instance): + return + def check_can_live_migrate_destination_cleanup(self, context, dest_check_data): return diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 488a33a2fb87..e119cbc74803 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -5815,6 +5815,23 @@ class LibvirtDriver(driver.ComputeDriver): post_method, recover_method, block_migration, migrate_data) + def live_migration_abort(self, instance): + """Aborting a running live-migration. + + :param instance: instance object that is in migration + + """ + + guest = self._host.get_guest(instance) + dom = guest._domain + + try: + dom.abortJob() + except libvirt.libvirtError as e: + LOG.error(_LE("Failed to cancel migration %s"), + e, instance=instance) + raise + def _update_xml(self, xml_str, migrate_bdm_info, listen_addrs, serial_listen_addr): xml_doc = etree.fromstring(xml_str) @@ -6428,7 +6445,7 @@ class LibvirtDriver(driver.ComputeDriver): LOG.warn(_LW("Migration operation was cancelled"), instance=instance) recover_method(context, instance, dest, block_migration, - migrate_data) + migrate_data, migration_status='cancelled') break else: LOG.warn(_LW("Unexpected migration job type: %d"), diff --git a/releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml b/releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml new file mode 100644 index 000000000000..f6e418ccf8ba --- /dev/null +++ b/releasenotes/notes/abort-live-migration-cb902bb0754b11b6.yaml @@ -0,0 +1,5 @@ +--- +features: + - A new REST API to cancel an ongoing live migration has been added + in microversion 2.24. Initially this operation will only work with + the libvirt virt driver.