Merge "Transform instance.rebuild.error notification"

doc/notification_samples/instance-rebuild-error.json

@@ -0,0 +1,84 @@
+{
+    "priority": "ERROR",
+    "payload": {
+        "nova_object.name": "InstanceActionPayload",
+        "nova_object.data": {
+            "state": "active",
+            "availability_zone": null,
+            "kernel_id": "",
+            "host_name": "some-server",
+            "progress": 0,
+            "task_state": "rebuilding",
+            "deleted_at": null,
+            "architecture": null,
+            "ramdisk_id": "",
+            "locked": false,
+            "created_at": "2012-10-29T13:42:11Z",
+            "host": "compute",
+            "display_name": "some-server",
+            "os_type": null,
+            "metadata": {},
+            "ip_addresses": [
+                {
+                    "nova_object.name": "IpPayload",
+                    "nova_object.data": {
+                        "device_name": "tapce531f90-19",
+                        "port_uuid": "ce531f90-199f-48c0-816c-13e38010b442",
+                        "address": "192.168.1.3",
+                        "version": 4,
+                        "meta": {},
+                        "label": "private-network",
+                        "mac": "fa:16:3e:4c:2c:30"
+                    },
+                    "nova_object.version": "1.0",
+                    "nova_object.namespace": "nova"
+                }
+            ],
+            "power_state": "running",
+            "display_description": "some-server",
+            "uuid": "5fafd989-4043-44b4-8acc-907e847f4b70",
+            "flavor": {
+                "nova_object.name": "FlavorPayload",
+                "nova_object.data": {
+                    "disabled": false,
+                    "ephemeral_gb": 0,
+                    "extra_specs": {"hw:watchdog_action": "disabled"},
+                    "flavorid": "a22d5517-147c-4147-a0d1-e698df5cd4e3",
+                    "is_public": true,
+                    "memory_mb": 512,
+                    "name": "test_flavor",
+                    "projects": null,
+                    "root_gb": 1,
+                    "rxtx_factor": 1.0,
+                    "swap": 0,
+                    "vcpu_weight": 0,
+                    "vcpus": 1
+                },
+                "nova_object.version": "1.3",
+                "nova_object.namespace": "nova"
+            },
+            "reservation_id": "r-pfiic52h",
+            "terminated_at": null,
+            "tenant_id": "6f70656e737461636b20342065766572",
+            "node": "fake-mini",
+            "launched_at": "2012-10-29T13:42:11Z",
+            "user_id": "fake",
+            "image_uuid": "a2459075-d96c-40d5-893e-577ff92e721c",
+            "fault": {
+                "nova_object.name": "ExceptionPayload",
+                "nova_object.data": {
+                    "module_name": "nova.tests.functional.notification_sample_tests.test_instance",
+                    "exception_message": "Insufficient compute resources: fake-resource.",
+                    "function_name": "_compute_resources_unavailable",
+                    "exception": "ComputeResourcesUnavailable"
+                },
+                "nova_object.version": "1.0",
+                "nova_object.namespace": "nova"
+            }
+        },
+        "nova_object.version": "1.1",
+        "nova_object.namespace": "nova"
+    },
+    "publisher_id": "nova-compute:compute",
+    "event_type": "instance.rebuild.error"
+}

nova/compute/manager.py

@@ -2668,6 +2668,14 @@ class ComputeManager(manager.Manager):
                               admin_password, network_info=network_info,
                               block_device_info=new_block_device_info)
 
+    def _notify_instance_rebuild_error(self, context, instance, error):
+        self._notify_about_instance_usage(context, instance,
+                                          'rebuild.error', fault=error)
+        compute_utils.notify_about_instance_action(
+            context, instance, self.host,
+            action=fields.NotificationAction.REBUILD,
+            phase=fields.NotificationPhase.ERROR, exception=error)
+
     @messaging.expected_exceptions(exception.PreserveEphemeralNotSupported)
     @wrap_exception()
     @reverts_task_state
@@ -2752,9 +2760,8 @@ class ComputeManager(manager.Manager):
                 # NOTE(ndipanov): We just abort the build for now and leave a
                 # migration record for potential cleanup later
                 self._set_migration_status(migration, 'failed')
+                self._notify_instance_rebuild_error(context, instance, e)
 
-                self._notify_about_instance_usage(context, instance,
-                        'rebuild.error', fault=e)
                 raise exception.BuildAbortException(
                     instance_uuid=instance.uuid, reason=e.format_message())
             except (exception.InstanceNotFound,
@@ -2762,12 +2769,10 @@ class ComputeManager(manager.Manager):
                 LOG.debug('Instance was deleted while rebuilding',
                           instance=instance)
                 self._set_migration_status(migration, 'failed')
-                self._notify_about_instance_usage(context, instance,
-                        'rebuild.error', fault=e)
+                self._notify_instance_rebuild_error(context, instance, e)
             except Exception as e:
                 self._set_migration_status(migration, 'failed')
-                self._notify_about_instance_usage(context, instance,
-                        'rebuild.error', fault=e)
+                self._notify_instance_rebuild_error(context, instance, e)
                 raise
             else:
                 instance.apply_migration_context()

nova/notifications/objects/instance.py

@@ -270,7 +270,7 @@ class InstanceStateUpdatePayload(base.NotificationPayloadBase):
 # @base.notification_sample('instance-live_migration_rollback_dest-end.json')
 @base.notification_sample('instance-rebuild-start.json')
 @base.notification_sample('instance-rebuild-end.json')
-# @base.notification_sample('instance-rebuild-error.json')
+@base.notification_sample('instance-rebuild-error.json')
 # @base.notification_sample('instance-remove_fixed_ip-start.json')
 # @base.notification_sample('instance-remove_fixed_ip-end.json')
 # @base.notification_sample('instance-resize_confirm-start.json')

nova/tests/functional/notification_sample_tests/test_instance.py

@@ -574,6 +574,35 @@ class TestInstanceNotificationSample(
                 'uuid': server['id']},
             actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
 
+    @mock.patch('nova.compute.manager.ComputeManager.'
+                '_do_rebuild_instance_with_claim')
+    def test_rebuild_server_exc(self, mock_rebuild):
+        def _compute_resources_unavailable(*args, **kwargs):
+            raise exception.ComputeResourcesUnavailable(
+                reason="fake-resource")
+
+        server = self._boot_a_server(
+            extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
+
+        fake_notifier.reset()
+
+        post = {
+            'rebuild': {
+                'imageRef': 'a2459075-d96c-40d5-893e-577ff92e721c',
+                'metadata': {}
+            }
+        }
+        self.api.post_server_action(server['id'], post)
+        mock_rebuild.side_effect = _compute_resources_unavailable
+        self._wait_for_state_change(self.api, server, expected_status='ERROR')
+        self.assertEqual(2, len(fake_notifier.VERSIONED_NOTIFICATIONS))
+        self._verify_notification(
+            'instance-rebuild-error',
+            replacements={
+                'reservation_id': server['reservation_id'],
+                'uuid': server['id']},
+            actual=fake_notifier.VERSIONED_NOTIFICATIONS[0])
+
     def _test_restore_server(self, server):
         self.flags(reclaim_instance_interval=30)
         self.api.delete_server(server['id'])

nova/tests/unit/compute/test_compute_mgr.py

@@ -3038,20 +3038,25 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase):
 
         do_test()
 
-    def _test_rebuild_ex(self, instance, ex):
-        # Test that we do not raise on certain exceptions
-        with test.nested(
-            mock.patch.object(self.compute, '_get_compute_info'),
-            mock.patch.object(self.compute, '_do_rebuild_instance_with_claim',
-                              side_effect=ex),
-            mock.patch.object(self.compute, '_set_migration_status'),
-            mock.patch.object(self.compute, '_notify_about_instance_usage')
-        ) as (mock_get, mock_rebuild, mock_set, mock_notify):
-            self.compute.rebuild_instance(self.context, instance, None, None,
-                                          None, None, None, None, None)
-            mock_set.assert_called_once_with(None, 'failed')
-            mock_notify.assert_called_once_with(mock.ANY, instance,
-                                                'rebuild.error', fault=ex)
+    @mock.patch.object(manager.ComputeManager, '_set_migration_status')
+    @mock.patch.object(manager.ComputeManager,
+                       '_do_rebuild_instance_with_claim')
+    @mock.patch('nova.compute.utils.notify_about_instance_action')
+    @mock.patch.object(manager.ComputeManager, '_notify_about_instance_usage')
+    def _test_rebuild_ex(self, instance, exc, mock_notify_about_instance_usage,
+                         mock_notify, mock_rebuild, mock_set):
+
+        mock_rebuild.side_effect = exc
+
+        self.compute.rebuild_instance(self.context, instance, None, None, None,
+                                      None, None, None, None)
+        mock_set.assert_called_once_with(None, 'failed')
+        mock_notify_about_instance_usage.assert_called_once_with(
+            mock.ANY, instance, 'rebuild.error', fault=mock_rebuild.side_effect
+        )
+        mock_notify.assert_called_once_with(
+            mock.ANY, instance, 'fake-mini', action='rebuild', phase='error',
+            exception=exc)
 
     def test_rebuild_deleting(self):
         instance = fake_instance.fake_instance_obj(self.context)