diff --git a/doc/notification_samples/instance-volume_attach-error.json b/doc/notification_samples/instance-volume_attach-error.json new file mode 100644 index 000000000000..425674a8494b --- /dev/null +++ b/doc/notification_samples/instance-volume_attach-error.json @@ -0,0 +1,86 @@ +{ + "event_type": "instance.volume_attach.error", + "payload": { + "nova_object.data": { + "architecture": null, + "availability_zone": "nova", + "created_at": "2012-10-29T13:42:11Z", + "deleted_at": null, + "display_name": "some-server", + "display_description":"some-server", + "fault": { + "nova_object.data": { + "exception": "CinderConnectionFailed", + "exception_message": "Connection to cinder host failed: Connection timed out", + "function_name": "attach_volume", + "module_name": "nova.tests.functional.notification_sample_tests.test_instance" + }, + "nova_object.name": "ExceptionPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0" + }, + "flavor": { + "nova_object.name": "FlavorPayload", + "nova_object.data": { + "flavorid": "a22d5517-147c-4147-a0d1-e698df5cd4e3", + "name": "test_flavor", + "projects": null, + "root_gb": 1, + "vcpus": 1, + "ephemeral_gb": 0, + "memory_mb": 512, + "disabled": false, + "rxtx_factor": 1.0, + "extra_specs": { + "hw:watchdog_action": "disabled" + }, + "swap": 0, + "is_public": true, + "vcpu_weight": 0 + }, + "nova_object.version": "1.3", + "nova_object.namespace": "nova" + }, + "host": "compute", + "host_name": "some-server", + "image_uuid": "a2459075-d96c-40d5-893e-577ff92e721c", + "ip_addresses": [{ + "nova_object.data": { + "address": "192.168.1.3", + "device_name": "tapce531f90-19", + "label": "private-network", + "meta": {}, + "port_uuid": "ce531f90-199f-48c0-816c-13e38010b442", + "version": 4, + "mac": "fa:16:3e:4c:2c:30" + }, + "nova_object.name": "IpPayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0" + }], + "kernel_id": "", + "launched_at": "2012-10-29T13:42:11Z", + "metadata": {}, + "locked":false, + "node": "fake-mini", + "volume_id": "a07f71dc-8151-4e7d-a0cc-cd24a3f11113", + "os_type": null, + "power_state":"running", + "progress": 0, + "ramdisk_id": "", + "reservation_id": "r-6w6ruqaz", + "state": "active", + "task_state": null, + "tenant_id": "6f70656e737461636b20342065766572", + "terminated_at": null, + "auto_disk_config": "MANUAL", + "user_id": "fake", + "uuid": "0ab886d0-7443-4107-9265-48371bfa662b" + }, + "nova_object.name": "InstanceActionVolumePayload", + "nova_object.namespace": "nova", + "nova_object.version": "1.0" + }, + "priority": "ERROR", + "publisher_id": "nova-compute:compute" +} diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 30d3a1b5bbcd..1df726c16e9a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -4819,7 +4819,7 @@ class ComputeManager(manager.Manager): try: bdm.attach(context, instance, self.volume_api, self.driver, do_driver_attach=True) - except Exception: + except Exception as e: with excutils.save_and_reraise_exception(): LOG.exception(_LE("Failed to attach %(volume_id)s " "at %(mountpoint)s"), @@ -4827,6 +4827,12 @@ class ComputeManager(manager.Manager): 'mountpoint': bdm['mount_device']}, instance=instance) self.volume_api.unreserve_volume(context, bdm.volume_id) + compute_utils.notify_about_volume_attach_detach( + context, instance, self.host, + action=fields.NotificationAction.VOLUME_ATTACH, + phase=fields.NotificationPhase.ERROR, + exception=e, + volume_id=bdm.volume_id) info = {'volume_id': bdm.volume_id} self._notify_about_instance_usage( diff --git a/nova/compute/utils.py b/nova/compute/utils.py index 1481a0eaa2d5..823949be9e57 100644 --- a/nova/compute/utils.py +++ b/nova/compute/utils.py @@ -367,7 +367,8 @@ def notify_about_instance_action(context, instance, host, action, phase=None, def notify_about_volume_attach_detach(context, instance, host, action, phase, - binary='nova-compute', volume_id=None): + binary='nova-compute', volume_id=None, + exception=None): """Send versioned notification about the action made on the instance :param instance: the instance which the action performed on :param host: the host emitting the notification @@ -375,14 +376,16 @@ def notify_about_volume_attach_detach(context, instance, host, action, phase, :param phase: the phase of the action :param binary: the binary emitting the notification :param volume_id: id of the volume will be attached + :param exception: the thrown exception (used in error notifications) """ + fault, priority = _get_fault_and_priority_from_exc(exception) payload = instance_notification.InstanceActionVolumePayload( instance=instance, - fault=None, + fault=fault, volume_id=volume_id) notification = instance_notification.InstanceActionVolumeNotification( context=context, - priority=fields.NotificationPriority.INFO, + priority=priority, publisher=notification_base.NotificationPublisher( context=context, host=host, binary=binary), event_type=notification_base.EventType( diff --git a/nova/notifications/objects/instance.py b/nova/notifications/objects/instance.py index 780fc86df138..6064b772796d 100644 --- a/nova/notifications/objects/instance.py +++ b/nova/notifications/objects/instance.py @@ -363,6 +363,7 @@ class InstanceActionVolumeSwapNotification(base.NotificationBase): @base.notification_sample('instance-volume_attach-start.json') @base.notification_sample('instance-volume_attach-end.json') +@base.notification_sample('instance-volume_attach-error.json') @nova_base.NovaObjectRegistry.register_notification class InstanceActionVolumeNotification(base.NotificationBase): # Version 1.0: Initial version diff --git a/nova/tests/functional/notification_sample_tests/test_instance.py b/nova/tests/functional/notification_sample_tests/test_instance.py index 978bd90787f7..c62d5f4aad33 100644 --- a/nova/tests/functional/notification_sample_tests/test_instance.py +++ b/nova/tests/functional/notification_sample_tests/test_instance.py @@ -83,6 +83,7 @@ class TestInstanceNotificationSample( self._test_rescue_server, self._test_unrescue_server, self._test_soft_delete_server, + self._test_attach_volume_error, ] for action in actions: @@ -800,3 +801,34 @@ class TestInstanceNotificationSample( def _test_soft_delete_server(self, server): pass + + @mock.patch('nova.volume.cinder.API.attach') + def _test_attach_volume_error(self, server, mock_attach): + def attach_volume(*args, **kwargs): + raise exception.CinderConnectionFailed( + reason="Connection timed out") + mock_attach.side_effect = attach_volume + + post = {"volumeAttachment": {"volumeId": self.cinder.SWAP_OLD_VOL}} + self.api.post_server_volume(server['id'], post) + + self._wait_for_notification('instance.volume_attach.error') + + # 0. volume_attach-start + # 1. volume_attach-error + # 2. compute.exception + # We only rely on the first 2 notifications, in this case we don't + # care about the exception notification. + self.assertLessEqual(2, len(fake_notifier.VERSIONED_NOTIFICATIONS)) + self._verify_notification( + 'instance-volume_attach-start', + replacements={ + 'reservation_id': server['reservation_id'], + 'uuid': server['id']}, + actual=fake_notifier.VERSIONED_NOTIFICATIONS[0]) + self._verify_notification( + 'instance-volume_attach-error', + replacements={ + 'reservation_id': server['reservation_id'], + 'uuid': server['id']}, + actual=fake_notifier.VERSIONED_NOTIFICATIONS[1]) diff --git a/nova/tests/unit/compute/test_compute.py b/nova/tests/unit/compute/test_compute.py index 11a5fe891775..723ef804a092 100644 --- a/nova/tests/unit/compute/test_compute.py +++ b/nova/tests/unit/compute/test_compute.py @@ -397,12 +397,17 @@ class ComputeVolumeTestCase(BaseTestCase): self.compute.attach_volume(self.context, instance, bdm=fake_bdm) self.assertEqual(self.cinfo.get('serial'), uuids.volume_id) - def test_attach_volume_raises(self): + @mock.patch('nova.context.RequestContext.elevated') + @mock.patch('nova.compute.utils.notify_about_volume_attach_detach') + def test_attach_volume_raises(self, mock_notify, mock_elevate): + mock_elevate.return_value = self.context + fake_bdm = objects.BlockDeviceMapping(**self.fake_volume) instance = self._create_fake_instance_obj() + expected_exception = test.TestingException() def fake_attach(*args, **kwargs): - raise test.TestingException + raise expected_exception with test.nested( mock.patch.object(driver_block_device.DriverVolumeBlockDevice, @@ -417,6 +422,15 @@ class ComputeVolumeTestCase(BaseTestCase): self.context, instance, fake_bdm) self.assertTrue(mock_unreserve.called) self.assertTrue(mock_destroy.called) + mock_notify.assert_has_calls([ + mock.call(self.context, instance, 'fake-mini', + action='volume_attach', phase='start', + volume_id=uuids.volume_id), + mock.call(self.context, instance, 'fake-mini', + action='volume_attach', phase='error', + volume_id=uuids.volume_id, + exception=expected_exception), + ]) def test_detach_volume_api_raises(self): fake_bdm = objects.BlockDeviceMapping(**self.fake_volume)