Merge "Add full traceback to ExceptionPayload in versioned notifications"
This commit is contained in:
commit
ecaadf6d6d
|
@ -5,11 +5,12 @@
|
|||
"exception": "AggregateNameExists",
|
||||
"exception_message": "Aggregate versioned_exc_aggregate already exists.",
|
||||
"function_name": "_aggregate_create_in_db",
|
||||
"module_name": "nova.objects.aggregate"
|
||||
"module_name": "nova.objects.aggregate",
|
||||
"traceback": "Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.name": "ExceptionPayload",
|
||||
"nova_object.namespace": "nova",
|
||||
"nova_object.version": "1.0"
|
||||
"nova_object.version": "1.1"
|
||||
},
|
||||
"priority": "ERROR",
|
||||
"publisher_id": "nova-api:fake-mini"
|
||||
|
|
|
@ -8,11 +8,12 @@
|
|||
"exception": "FlavorDiskTooSmall",
|
||||
"exception_message": "The created instance's disk would be too small.",
|
||||
"function_name": "_build_resources",
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance"
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance",
|
||||
"traceback": "Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.name": "ExceptionPayload",
|
||||
"nova_object.namespace": "nova",
|
||||
"nova_object.version": "1.0"
|
||||
"nova_object.version": "1.1"
|
||||
},
|
||||
"ip_addresses": [],
|
||||
"launched_at": null,
|
||||
|
|
|
@ -9,11 +9,12 @@
|
|||
"exception": "InterfaceAttachFailed",
|
||||
"exception_message": "dummy",
|
||||
"function_name": "_unsuccessful_attach_interface",
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance"
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance",
|
||||
"traceback": "Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.name": "ExceptionPayload",
|
||||
"nova_object.namespace": "nova",
|
||||
"nova_object.version": "1.0"
|
||||
"nova_object.version": "1.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
@ -8,11 +8,12 @@
|
|||
"exception": "UnsupportedVirtType",
|
||||
"exception_message": "Virtualization type 'FakeVirt' is not supported by this compute driver",
|
||||
"function_name": "_hard_reboot",
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance"
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance",
|
||||
"traceback": "Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.name": "ExceptionPayload",
|
||||
"nova_object.namespace": "nova",
|
||||
"nova_object.version": "1.0"
|
||||
"nova_object.version": "1.1"
|
||||
},
|
||||
"task_state":"reboot_started_hard"
|
||||
}
|
||||
|
|
|
@ -9,9 +9,10 @@
|
|||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance",
|
||||
"exception_message": "Virtual Interface creation failed",
|
||||
"function_name": "_virtual_interface_create_failed",
|
||||
"exception": "VirtualInterfaceCreateException"
|
||||
"exception": "VirtualInterfaceCreateException",
|
||||
"traceback": "Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.version": "1.0",
|
||||
"nova_object.version": "1.1",
|
||||
"nova_object.namespace": "nova"
|
||||
},
|
||||
"architecture": null,
|
||||
|
|
|
@ -8,11 +8,12 @@
|
|||
"exception":"FlavorDiskTooSmall",
|
||||
"exception_message":"The created instance's disk would be too small.",
|
||||
"function_name":"_build_resources",
|
||||
"module_name":"nova.tests.functional.notification_sample_tests.test_instance"
|
||||
"module_name":"nova.tests.functional.notification_sample_tests.test_instance",
|
||||
"traceback":"Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.name":"ExceptionPayload",
|
||||
"nova_object.namespace":"nova",
|
||||
"nova_object.version":"1.0"
|
||||
"nova_object.version":"1.1"
|
||||
},
|
||||
"block_devices": [],
|
||||
"task_state": "resize_prep"
|
||||
|
|
|
@ -8,11 +8,12 @@
|
|||
"exception": "CinderConnectionFailed",
|
||||
"exception_message": "Connection to cinder host failed: Connection timed out",
|
||||
"function_name": "attach_volume",
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance"
|
||||
"module_name": "nova.tests.functional.notification_sample_tests.test_instance",
|
||||
"traceback": "Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.name": "ExceptionPayload",
|
||||
"nova_object.namespace": "nova",
|
||||
"nova_object.version": "1.0"
|
||||
"nova_object.version": "1.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
@ -10,11 +10,12 @@
|
|||
"exception": "TypeError",
|
||||
"exception_message": "'tuple' object does not support item assignment",
|
||||
"function_name": "_init_volume_connection",
|
||||
"module_name": "nova.compute.manager"
|
||||
"module_name": "nova.compute.manager",
|
||||
"traceback": "Traceback (most recent call last):\n File \"nova/compute/manager.py\", line ..."
|
||||
},
|
||||
"nova_object.name": "ExceptionPayload",
|
||||
"nova_object.namespace": "nova",
|
||||
"nova_object.version": "1.0"
|
||||
"nova_object.version": "1.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
@ -2017,18 +2017,20 @@ class ComputeManager(manager.Manager):
|
|||
with excutils.save_and_reraise_exception():
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
except exception.ComputeResourcesUnavailable as e:
|
||||
LOG.debug(e.format_message(), instance=instance)
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
raise exception.RescheduledException(
|
||||
instance_uuid=instance.uuid, reason=e.format_message())
|
||||
except exception.BuildAbortException as e:
|
||||
|
@ -2036,20 +2038,22 @@ class ComputeManager(manager.Manager):
|
|||
LOG.debug(e.format_message(), instance=instance)
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
except (exception.FixedIpLimitExceeded,
|
||||
exception.NoMoreNetworks, exception.NoMoreFixedIps) as e:
|
||||
LOG.warning('No more network or fixed IP to be allocated',
|
||||
instance=instance)
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
msg = _('Failed to allocate the network(s) with error %s, '
|
||||
'not rescheduling.') % e.format_message()
|
||||
raise exception.BuildAbortException(instance_uuid=instance.uuid,
|
||||
|
@ -2062,10 +2066,11 @@ class ComputeManager(manager.Manager):
|
|||
instance=instance)
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
msg = _('Failed to allocate the network(s), not rescheduling.')
|
||||
raise exception.BuildAbortException(instance_uuid=instance.uuid,
|
||||
reason=msg)
|
||||
|
@ -2084,19 +2089,21 @@ class ComputeManager(manager.Manager):
|
|||
exception.RequestedVRamTooHigh) as e:
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
raise exception.BuildAbortException(instance_uuid=instance.uuid,
|
||||
reason=e.format_message())
|
||||
except Exception as e:
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
raise exception.RescheduledException(
|
||||
instance_uuid=instance.uuid, reason=six.text_type(e))
|
||||
|
||||
|
@ -2128,10 +2135,11 @@ class ComputeManager(manager.Manager):
|
|||
with excutils.save_and_reraise_exception():
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'create.error', fault=e)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_create(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=e,
|
||||
bdms=block_device_mapping)
|
||||
bdms=block_device_mapping, tb=tb)
|
||||
|
||||
self._update_scheduler_instance_info(context, instance)
|
||||
self._notify_about_instance_usage(context, instance, 'create.end',
|
||||
|
@ -2775,11 +2783,13 @@ class ComputeManager(manager.Manager):
|
|||
block_device_info=new_block_device_info)
|
||||
|
||||
def _notify_instance_rebuild_error(self, context, instance, error, bdms):
|
||||
tb = traceback.format_exc()
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'rebuild.error', fault=error)
|
||||
compute_utils.notify_about_instance_rebuild(
|
||||
context, instance, self.host,
|
||||
phase=fields.NotificationPhase.ERROR, exception=error, bdms=bdms)
|
||||
phase=fields.NotificationPhase.ERROR, exception=error, bdms=bdms,
|
||||
tb=tb)
|
||||
|
||||
@messaging.expected_exceptions(exception.PreserveEphemeralNotSupported)
|
||||
@wrap_exception()
|
||||
|
@ -3229,11 +3239,12 @@ class ComputeManager(manager.Manager):
|
|||
instance, error, exc_info)
|
||||
self._notify_about_instance_usage(context, instance,
|
||||
'reboot.error', fault=error)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_action(
|
||||
context, instance, self.host,
|
||||
action=fields.NotificationAction.REBOOT,
|
||||
phase=fields.NotificationPhase.ERROR,
|
||||
exception=error, bdms=bdms
|
||||
exception=error, bdms=bdms, tb=tb
|
||||
)
|
||||
ctxt.reraise = False
|
||||
else:
|
||||
|
@ -4202,7 +4213,8 @@ class ComputeManager(manager.Manager):
|
|||
context, instance, self.host,
|
||||
action=fields.NotificationAction.RESIZE,
|
||||
phase=fields.NotificationPhase.ERROR,
|
||||
exception=error)
|
||||
exception=error,
|
||||
tb=','.join(traceback.format_exception(*exc_info)))
|
||||
if rescheduled:
|
||||
self._log_original_error(exc_info, instance_uuid)
|
||||
compute_utils.add_instance_fault_from_exc(context,
|
||||
|
@ -4213,7 +4225,8 @@ class ComputeManager(manager.Manager):
|
|||
context, instance, self.host,
|
||||
action=fields.NotificationAction.RESIZE,
|
||||
phase=fields.NotificationPhase.ERROR,
|
||||
exception=exc_info[1])
|
||||
exception=exc_info[1],
|
||||
tb=','.join(traceback.format_exception(*exc_info)))
|
||||
else:
|
||||
# not re-scheduling
|
||||
six.reraise(*exc_info)
|
||||
|
@ -5360,12 +5373,13 @@ class ComputeManager(manager.Manager):
|
|||
bdm['attachment_id'])
|
||||
else:
|
||||
self.volume_api.unreserve_volume(context, bdm.volume_id)
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_volume_attach_detach(
|
||||
context, instance, self.host,
|
||||
action=fields.NotificationAction.VOLUME_ATTACH,
|
||||
phase=fields.NotificationPhase.ERROR,
|
||||
exception=e,
|
||||
volume_id=bdm.volume_id)
|
||||
volume_id=bdm.volume_id, tb=tb)
|
||||
|
||||
info = {'volume_id': bdm.volume_id}
|
||||
self._notify_about_instance_usage(
|
||||
|
@ -5561,10 +5575,11 @@ class ComputeManager(manager.Manager):
|
|||
except Exception as ex:
|
||||
failed = True
|
||||
with excutils.save_and_reraise_exception():
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_volume_swap(
|
||||
context, instance, self.host,
|
||||
fields.NotificationPhase.ERROR,
|
||||
old_volume_id, new_volume_id, ex)
|
||||
old_volume_id, new_volume_id, ex, tb)
|
||||
if new_cinfo:
|
||||
msg = ("Failed to swap volume %(old_volume_id)s "
|
||||
"for %(new_volume_id)s")
|
||||
|
@ -5799,11 +5814,12 @@ class ComputeManager(manager.Manager):
|
|||
LOG.warning("deallocate port %(port_id)s failed",
|
||||
{'port_id': port_id}, instance=instance)
|
||||
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_instance_action(
|
||||
context, instance, self.host,
|
||||
action=fields.NotificationAction.INTERFACE_ATTACH,
|
||||
phase=fields.NotificationPhase.ERROR,
|
||||
exception=ex)
|
||||
exception=ex, tb=tb)
|
||||
|
||||
raise exception.InterfaceAttachFailed(
|
||||
instance_uuid=instance.uuid)
|
||||
|
|
|
@ -366,14 +366,14 @@ def notify_about_instance_usage(notifier, context, instance, event_suffix,
|
|||
method(context, 'compute.instance.%s' % event_suffix, usage_info)
|
||||
|
||||
|
||||
def _get_fault_and_priority_from_exc(exception):
|
||||
def _get_fault_and_priority_from_exc_and_tb(exception, tb):
|
||||
fault = None
|
||||
priority = fields.NotificationPriority.INFO
|
||||
|
||||
if exception:
|
||||
priority = fields.NotificationPriority.ERROR
|
||||
fault = notification_exception.ExceptionPayload.from_exception(
|
||||
exception)
|
||||
fault = notification_exception.ExceptionPayload.from_exc_and_traceback(
|
||||
exception, tb)
|
||||
|
||||
return fault, priority
|
||||
|
||||
|
@ -381,7 +381,7 @@ def _get_fault_and_priority_from_exc(exception):
|
|||
@rpc.if_notifications_enabled
|
||||
def notify_about_instance_action(context, instance, host, action, phase=None,
|
||||
source=fields.NotificationSource.COMPUTE,
|
||||
exception=None, bdms=None):
|
||||
exception=None, bdms=None, tb=None):
|
||||
"""Send versioned notification about the action made on the instance
|
||||
:param instance: the instance which the action performed on
|
||||
:param host: the host emitting the notification
|
||||
|
@ -391,8 +391,9 @@ def notify_about_instance_action(context, instance, host, action, phase=None,
|
|||
:param exception: the thrown exception (used in error notifications)
|
||||
:param bdms: BlockDeviceMappingList object for the instance. If it is not
|
||||
provided then we will load it from the db if so configured
|
||||
:param tb: the traceback (used in error notifications)
|
||||
"""
|
||||
fault, priority = _get_fault_and_priority_from_exc(exception)
|
||||
fault, priority = _get_fault_and_priority_from_exc_and_tb(exception, tb)
|
||||
payload = instance_notification.InstanceActionPayload(
|
||||
context=context,
|
||||
instance=instance,
|
||||
|
@ -413,7 +414,7 @@ def notify_about_instance_action(context, instance, host, action, phase=None,
|
|||
|
||||
@rpc.if_notifications_enabled
|
||||
def notify_about_instance_create(context, instance, host, phase=None,
|
||||
exception=None, bdms=None):
|
||||
exception=None, bdms=None, tb=None):
|
||||
"""Send versioned notification about instance creation
|
||||
|
||||
:param context: the request context
|
||||
|
@ -423,8 +424,9 @@ def notify_about_instance_create(context, instance, host, phase=None,
|
|||
:param exception: the thrown exception (used in error notifications)
|
||||
:param bdms: BlockDeviceMappingList object for the instance. If it is not
|
||||
provided then we will load it from the db if so configured
|
||||
:param tb: the traceback (used in error notifications)
|
||||
"""
|
||||
fault, priority = _get_fault_and_priority_from_exc(exception)
|
||||
fault, priority = _get_fault_and_priority_from_exc_and_tb(exception, tb)
|
||||
payload = instance_notification.InstanceCreatePayload(
|
||||
context=context,
|
||||
instance=instance,
|
||||
|
@ -445,7 +447,7 @@ def notify_about_instance_create(context, instance, host, phase=None,
|
|||
|
||||
@rpc.if_notifications_enabled
|
||||
def notify_about_volume_attach_detach(context, instance, host, action, phase,
|
||||
volume_id=None, exception=None):
|
||||
volume_id=None, exception=None, tb=None):
|
||||
"""Send versioned notification about the action made on the instance
|
||||
:param instance: the instance which the action performed on
|
||||
:param host: the host emitting the notification
|
||||
|
@ -453,8 +455,9 @@ def notify_about_volume_attach_detach(context, instance, host, action, phase,
|
|||
:param phase: the phase of the action
|
||||
:param volume_id: id of the volume will be attached
|
||||
:param exception: the thrown exception (used in error notifications)
|
||||
:param tb: the traceback (used in error notifications)
|
||||
"""
|
||||
fault, priority = _get_fault_and_priority_from_exc(exception)
|
||||
fault, priority = _get_fault_and_priority_from_exc_and_tb(exception, tb)
|
||||
payload = instance_notification.InstanceActionVolumePayload(
|
||||
context=context,
|
||||
instance=instance,
|
||||
|
@ -474,8 +477,9 @@ def notify_about_volume_attach_detach(context, instance, host, action, phase,
|
|||
|
||||
|
||||
@rpc.if_notifications_enabled
|
||||
def notify_about_instance_rescue_action(
|
||||
context, instance, host, rescue_image_ref, phase=None, exception=None):
|
||||
def notify_about_instance_rescue_action(context, instance, host,
|
||||
rescue_image_ref, phase=None,
|
||||
exception=None, tb=None):
|
||||
"""Send versioned notification about the action made on the instance
|
||||
|
||||
:param instance: the instance which the action performed on
|
||||
|
@ -483,8 +487,9 @@ def notify_about_instance_rescue_action(
|
|||
:param rescue_image_ref: the rescue image ref
|
||||
:param phase: the phase of the action
|
||||
:param exception: the thrown exception (used in error notifications)
|
||||
:param tb: the traceback (used in error notifications)
|
||||
"""
|
||||
fault, priority = _get_fault_and_priority_from_exc(exception)
|
||||
fault, priority = _get_fault_and_priority_from_exc_and_tb(exception, tb)
|
||||
payload = instance_notification.InstanceActionRescuePayload(
|
||||
context=context,
|
||||
instance=instance,
|
||||
|
@ -528,7 +533,8 @@ def notify_about_keypair_action(context, keypair, action, phase):
|
|||
|
||||
@rpc.if_notifications_enabled
|
||||
def notify_about_volume_swap(context, instance, host, phase,
|
||||
old_volume_id, new_volume_id, exception=None):
|
||||
old_volume_id, new_volume_id, exception=None,
|
||||
tb=None):
|
||||
"""Send versioned notification about the volume swap action
|
||||
on the instance
|
||||
|
||||
|
@ -539,8 +545,9 @@ def notify_about_volume_swap(context, instance, host, phase,
|
|||
:param old_volume_id: the ID of the volume that is copied from and detached
|
||||
:param new_volume_id: the ID of the volume that is copied to and attached
|
||||
:param exception: an exception
|
||||
:param tb: the traceback (used in error notifications)
|
||||
"""
|
||||
fault, priority = _get_fault_and_priority_from_exc(exception)
|
||||
fault, priority = _get_fault_and_priority_from_exc_and_tb(exception, tb)
|
||||
payload = instance_notification.InstanceActionVolumeSwapPayload(
|
||||
context=context,
|
||||
instance=instance,
|
||||
|
@ -717,7 +724,7 @@ def notify_about_server_group_add_member(context, group_id):
|
|||
|
||||
@rpc.if_notifications_enabled
|
||||
def notify_about_instance_rebuild(context, instance, host, phase=None,
|
||||
exception=None, bdms=None):
|
||||
exception=None, bdms=None, tb=None):
|
||||
"""Send versioned notification about instance rebuild
|
||||
|
||||
:param instance: the instance which the action performed on
|
||||
|
@ -726,8 +733,9 @@ def notify_about_instance_rebuild(context, instance, host, phase=None,
|
|||
:param exception: the thrown exception (used in error notifications)
|
||||
:param bdms: BlockDeviceMappingList object for the instance. If it is not
|
||||
provided then we will load it from the db if so configured
|
||||
:param tb: the traceback (used in error notifications)
|
||||
"""
|
||||
fault, priority = _get_fault_and_priority_from_exc(exception)
|
||||
fault, priority = _get_fault_and_priority_from_exc_and_tb(exception, tb)
|
||||
payload = instance_notification.InstanceActionRebuildPayload(
|
||||
context=context,
|
||||
instance=instance,
|
||||
|
|
|
@ -12,6 +12,7 @@
|
|||
|
||||
import functools
|
||||
import inspect
|
||||
import traceback
|
||||
|
||||
from oslo_utils import excutils
|
||||
|
||||
|
@ -27,15 +28,16 @@ CONF = nova.conf.CONF
|
|||
|
||||
|
||||
def _emit_exception_notification(notifier, context, ex, function_name, args,
|
||||
source):
|
||||
source, trace_back):
|
||||
_emit_legacy_exception_notification(notifier, context, ex, function_name,
|
||||
args)
|
||||
_emit_versioned_exception_notification(context, ex, source)
|
||||
_emit_versioned_exception_notification(context, ex, source, trace_back)
|
||||
|
||||
|
||||
@rpc.if_notifications_enabled
|
||||
def _emit_versioned_exception_notification(context, ex, source):
|
||||
versioned_exception_payload = exception.ExceptionPayload.from_exception(ex)
|
||||
def _emit_versioned_exception_notification(context, ex, source, trace_back):
|
||||
versioned_exception_payload = \
|
||||
exception.ExceptionPayload.from_exc_and_traceback(ex, trace_back)
|
||||
publisher = base.NotificationPublisher(host=CONF.host, source=source)
|
||||
event_type = base.EventType(
|
||||
object='compute',
|
||||
|
@ -66,6 +68,7 @@ def wrap_exception(notifier=None, get_notifier=None, binary=None):
|
|||
try:
|
||||
return f(self, context, *args, **kw)
|
||||
except Exception as e:
|
||||
tb = traceback.format_exc()
|
||||
with excutils.save_and_reraise_exception():
|
||||
if notifier or get_notifier:
|
||||
call_dict = _get_call_dict(
|
||||
|
@ -73,7 +76,7 @@ def wrap_exception(notifier=None, get_notifier=None, binary=None):
|
|||
function_name = f.__name__
|
||||
_emit_exception_notification(
|
||||
notifier or get_notifier(), context, e,
|
||||
function_name, call_dict, binary)
|
||||
function_name, call_dict, binary, tb)
|
||||
|
||||
return functools.wraps(f)(wrapped)
|
||||
return inner
|
||||
|
|
|
@ -21,24 +21,27 @@ from nova.objects import fields
|
|||
@nova_base.NovaObjectRegistry.register_notification
|
||||
class ExceptionPayload(base.NotificationPayloadBase):
|
||||
# Version 1.0: Initial version
|
||||
VERSION = '1.0'
|
||||
# Version 1.1: Add traceback field to ExceptionPayload
|
||||
VERSION = '1.1'
|
||||
fields = {
|
||||
'module_name': fields.StringField(),
|
||||
'function_name': fields.StringField(),
|
||||
'exception': fields.StringField(),
|
||||
'exception_message': fields.StringField()
|
||||
'exception_message': fields.StringField(),
|
||||
'traceback': fields.StringField()
|
||||
}
|
||||
|
||||
def __init__(self, module_name, function_name, exception,
|
||||
exception_message):
|
||||
exception_message, traceback):
|
||||
super(ExceptionPayload, self).__init__()
|
||||
self.module_name = module_name
|
||||
self.function_name = function_name
|
||||
self.exception = exception
|
||||
self.exception_message = exception_message
|
||||
self.traceback = traceback
|
||||
|
||||
@classmethod
|
||||
def from_exception(cls, fault):
|
||||
def from_exc_and_traceback(cls, fault, traceback):
|
||||
trace = inspect.trace()[-1]
|
||||
# TODO(gibi): apply strutils.mask_password on exception_message and
|
||||
# consider emitting the exception_message only if the safe flag is
|
||||
|
@ -49,7 +52,8 @@ class ExceptionPayload(base.NotificationPayloadBase):
|
|||
function_name=trace[3],
|
||||
module_name=module_name,
|
||||
exception=fault.__class__.__name__,
|
||||
exception_message=six.text_type(fault))
|
||||
exception_message=six.text_type(fault),
|
||||
traceback=traceback)
|
||||
|
||||
|
||||
@base.notification_sample('compute-exception.json')
|
||||
|
|
|
@ -34,6 +34,11 @@ class TestExceptionNotificationSample(
|
|||
self.admin_api.api_post, 'os-aggregates', post)
|
||||
|
||||
self.assertEqual(4, len(fake_notifier.VERSIONED_NOTIFICATIONS))
|
||||
traceback = fake_notifier.VERSIONED_NOTIFICATIONS[3][
|
||||
'payload']['nova_object.data']['traceback']
|
||||
self.assertIn('AggregateNameExists', traceback)
|
||||
self._verify_notification(
|
||||
'compute-exception',
|
||||
replacements={
|
||||
'traceback': self.ANY},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[3])
|
||||
|
|
|
@ -367,6 +367,10 @@ class TestInstanceNotificationSample(
|
|||
|
||||
self.assertEqual(2, len(fake_notifier.VERSIONED_NOTIFICATIONS))
|
||||
|
||||
tb = fake_notifier.VERSIONED_NOTIFICATIONS[1]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn('raise exception.FlavorDiskTooSmall()', tb)
|
||||
|
||||
self._verify_notification(
|
||||
'instance-create-start',
|
||||
replacements={
|
||||
|
@ -377,7 +381,8 @@ class TestInstanceNotificationSample(
|
|||
'instance-create-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id']},
|
||||
'uuid': server['id'],
|
||||
'fault.traceback': self.ANY},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
|
||||
|
||||
def test_instance_exists_usage_audit(self):
|
||||
|
@ -910,10 +915,16 @@ class TestInstanceNotificationSample(
|
|||
fake_notifier.VERSIONED_NOTIFICATIONS)
|
||||
# Note(gibi): There is also an instance.exists notification emitted
|
||||
# during the rescheduling
|
||||
|
||||
tb = fake_notifier.VERSIONED_NOTIFICATIONS[2]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn("raise exception.FlavorDiskTooSmall()", tb)
|
||||
|
||||
self._verify_notification('instance-resize-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id']
|
||||
'uuid': server['id'],
|
||||
'fault.traceback': self.ANY
|
||||
},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[2])
|
||||
|
||||
|
@ -970,10 +981,16 @@ class TestInstanceNotificationSample(
|
|||
self.assertEqual(5, len(fake_notifier.VERSIONED_NOTIFICATIONS),
|
||||
'Unexpected number of notifications: %s' %
|
||||
fake_notifier.VERSIONED_NOTIFICATIONS)
|
||||
|
||||
tb = fake_notifier.VERSIONED_NOTIFICATIONS[2]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn("raise exception.FlavorDiskTooSmall()", tb)
|
||||
|
||||
self._verify_notification('instance-resize-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id']
|
||||
'uuid': server['id'],
|
||||
'fault.traceback': self.ANY
|
||||
},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[2])
|
||||
|
||||
|
@ -1146,12 +1163,18 @@ class TestInstanceNotificationSample(
|
|||
self._wait_for_state_change(self.api, server, expected_status='ERROR')
|
||||
notification = self._get_notifications('instance.rebuild.error')
|
||||
self.assertEqual(1, len(notification))
|
||||
|
||||
tb = fake_notifier.VERSIONED_NOTIFICATIONS[0]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn('raise exception.VirtualInterfaceCreateException()', tb)
|
||||
|
||||
self._verify_notification(
|
||||
'instance-rebuild-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id'],
|
||||
'trusted_image_certificates': None},
|
||||
'trusted_image_certificates': None,
|
||||
'fault.traceback': self.ANY},
|
||||
actual=notification[0])
|
||||
|
||||
def _test_restore_server(self, server):
|
||||
|
@ -1207,6 +1230,11 @@ class TestInstanceNotificationSample(
|
|||
self._wait_for_notification('instance.reboot.start')
|
||||
self._wait_for_notification('instance.reboot.error')
|
||||
self.assertEqual(2, len(fake_notifier.VERSIONED_NOTIFICATIONS))
|
||||
|
||||
tb = fake_notifier.VERSIONED_NOTIFICATIONS[1]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn("raise exception.UnsupportedVirtType", tb)
|
||||
|
||||
self._verify_notification(
|
||||
'instance-reboot-start',
|
||||
replacements={
|
||||
|
@ -1217,7 +1245,8 @@ class TestInstanceNotificationSample(
|
|||
'instance-reboot-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id']},
|
||||
'uuid': server['id'],
|
||||
'fault.traceback': self.ANY},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
|
||||
|
||||
def _detach_volume_from_server(self, server, volume_id):
|
||||
|
@ -1312,12 +1341,21 @@ class TestInstanceNotificationSample(
|
|||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id']},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[5])
|
||||
|
||||
tb1 = fake_notifier.VERSIONED_NOTIFICATIONS[6]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn("_swap_volume", tb1)
|
||||
tb2 = fake_notifier.VERSIONED_NOTIFICATIONS[7]['payload'][
|
||||
'nova_object.data']['traceback']
|
||||
self.assertIn("_swap_volume", tb2)
|
||||
|
||||
self._verify_notification(
|
||||
'instance-volume_swap-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'block_devices': block_devices,
|
||||
'uuid': server['id']},
|
||||
'uuid': server['id'],
|
||||
'fault.traceback': self.ANY},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[6])
|
||||
|
||||
def test_resize_confirm_server(self):
|
||||
|
@ -1525,13 +1563,19 @@ class TestInstanceNotificationSample(
|
|||
'volume_id': self.cinder.SWAP_NEW_VOL,
|
||||
'uuid': server['id']},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[0])
|
||||
|
||||
tb = fake_notifier.VERSIONED_NOTIFICATIONS[1]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn("CinderConnectionFailed:", tb)
|
||||
|
||||
self._verify_notification(
|
||||
'instance-volume_attach-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'block_devices': block_devices,
|
||||
'volume_id': self.cinder.SWAP_NEW_VOL,
|
||||
'uuid': server['id']},
|
||||
'uuid': server['id'],
|
||||
'fault.traceback': self.ANY},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
|
||||
|
||||
@mock.patch('nova.volume.cinder.API.attachment_update')
|
||||
|
@ -1605,11 +1649,17 @@ class TestInstanceNotificationSample(
|
|||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id']},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[0])
|
||||
|
||||
tb = fake_notifier.VERSIONED_NOTIFICATIONS[1]['payload'][
|
||||
'nova_object.data']['fault']['nova_object.data']['traceback']
|
||||
self.assertIn("raise exception.InterfaceAttachFailed", tb)
|
||||
|
||||
self._verify_notification(
|
||||
'instance-interface_attach-error',
|
||||
replacements={
|
||||
'reservation_id': server['reservation_id'],
|
||||
'uuid': server['id']},
|
||||
'uuid': server['id'],
|
||||
'fault.traceback': self.ANY},
|
||||
actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
|
||||
|
||||
|
||||
|
|
|
@ -428,7 +428,8 @@ class ComputeVolumeTestCase(BaseTestCase):
|
|||
mock.call(self.context, instance, 'fake-mini',
|
||||
action='volume_attach', phase='error',
|
||||
volume_id=uuids.volume_id,
|
||||
exception=expected_exception),
|
||||
exception=expected_exception,
|
||||
tb=mock.ANY),
|
||||
])
|
||||
mock_event.assert_called_once_with(
|
||||
self.context, 'compute_attach_volume', CONF.host,
|
||||
|
@ -468,7 +469,8 @@ class ComputeVolumeTestCase(BaseTestCase):
|
|||
mock.call(self.context, instance, 'fake-mini',
|
||||
action='volume_attach', phase='error',
|
||||
volume_id=uuids.volume_id,
|
||||
exception=expected_exception),
|
||||
exception=expected_exception,
|
||||
tb=mock.ANY),
|
||||
])
|
||||
|
||||
@mock.patch.object(compute_utils, 'EventReporter')
|
||||
|
@ -3182,7 +3184,7 @@ class ComputeTestCase(BaseTestCase,
|
|||
notify_action_call_list.append(
|
||||
mock.call(econtext, instance, 'fake-mini',
|
||||
action='reboot', phase='error', exception=fault,
|
||||
bdms=bdms))
|
||||
bdms=bdms, tb=mock.ANY))
|
||||
notify_call_list.append(mock.call(econtext, instance,
|
||||
'reboot.end'))
|
||||
notify_action_call_list.append(
|
||||
|
@ -10253,7 +10255,7 @@ class ComputeAPITestCase(BaseTestCase):
|
|||
mock.call(self.context, instance, self.compute.host,
|
||||
action='interface_attach',
|
||||
exception=mock_attach.side_effect,
|
||||
phase='error')])
|
||||
phase='error', tb=mock.ANY)])
|
||||
|
||||
@mock.patch.object(compute_utils, 'notify_about_instance_action')
|
||||
def test_detach_interface(self, mock_notify):
|
||||
|
@ -12529,7 +12531,7 @@ class ComputeRescheduleResizeOrReraiseTestCase(BaseTestCase):
|
|||
task_states.RESIZE_PREP, exc_info, host_list=None)
|
||||
mock_notify.assert_called_once_with(
|
||||
self.context, instance, 'fake-mini', action='resize',
|
||||
phase='error', exception=mock_res.side_effect)
|
||||
phase='error', exception=mock_res.side_effect, tb=mock.ANY)
|
||||
|
||||
@mock.patch.object(compute_manager.ComputeManager, "_reschedule")
|
||||
def test_reschedule_false(self, mock_res):
|
||||
|
|
|
@ -2031,7 +2031,7 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase):
|
|||
self.compute.host,
|
||||
fields.NotificationPhase.ERROR,
|
||||
uuids.old_volume, uuids.new_volume,
|
||||
test.MatchType(expected_exception))
|
||||
test.MatchType(expected_exception), mock.ANY)
|
||||
else:
|
||||
self.compute.swap_volume(self.context, uuids.old_volume,
|
||||
uuids.new_volume, instance1, None)
|
||||
|
@ -3987,7 +3987,7 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase):
|
|||
)
|
||||
mock_notify.assert_called_once_with(
|
||||
mock.ANY, instance, 'fake-mini', phase='error', exception=exc,
|
||||
bdms=None)
|
||||
bdms=None, tb=mock.ANY)
|
||||
|
||||
def test_rebuild_deleting(self):
|
||||
instance = fake_instance.fake_instance_obj(self.context)
|
||||
|
@ -4115,7 +4115,7 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase):
|
|||
elevated_context, instance, 'fake-node', node_type='destination')
|
||||
mock_notify.assert_called_once_with(
|
||||
elevated_context, instance, 'fake-mini', bdms=None, exception=exc,
|
||||
phase='error')
|
||||
phase='error', tb=mock.ANY)
|
||||
|
||||
def test_rebuild_node_not_updated_if_not_recreate(self):
|
||||
node = uuidutils.generate_uuid() # ironic node uuid
|
||||
|
@ -5567,7 +5567,7 @@ class ComputeManagerBuildInstanceTestCase(test.NoDBTestCase):
|
|||
mock.call(self.context, self.instance, 'fake-mini',
|
||||
phase='start', bdms=[]),
|
||||
mock.call(self.context, self.instance, 'fake-mini',
|
||||
phase='error', exception=exc, bdms=[])])
|
||||
phase='error', exception=exc, bdms=[], tb=mock.ANY)])
|
||||
|
||||
save.assert_has_calls([
|
||||
mock.call(),
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
import copy
|
||||
import string
|
||||
import traceback
|
||||
|
||||
import mock
|
||||
from oslo_serialization import jsonutils
|
||||
|
@ -733,10 +734,11 @@ class UsageInfoTestCase(test.TestCase):
|
|||
# To get exception trace, raise and catch an exception
|
||||
raise test.TestingException('Volume swap error.')
|
||||
except Exception as ex:
|
||||
tb = traceback.format_exc()
|
||||
compute_utils.notify_about_volume_swap(
|
||||
self.context, instance, 'fake-compute',
|
||||
fields.NotificationPhase.ERROR,
|
||||
uuids.old_volume_id, uuids.new_volume_id, ex)
|
||||
uuids.old_volume_id, uuids.new_volume_id, ex, tb)
|
||||
|
||||
self.assertEqual(len(fake_notifier.VERSIONED_NOTIFICATIONS), 1)
|
||||
notification = fake_notifier.VERSIONED_NOTIFICATIONS[0]
|
||||
|
@ -776,6 +778,8 @@ class UsageInfoTestCase(test.TestCase):
|
|||
exception_payload['function_name'])
|
||||
self.assertEqual('nova.tests.unit.compute.test_compute_utils',
|
||||
exception_payload['module_name'])
|
||||
self.assertIn('test_notify_about_volume_swap_with_error',
|
||||
exception_payload['traceback'])
|
||||
|
||||
def test_notify_about_instance_rescue_action(self):
|
||||
instance = create_instance(self.context)
|
||||
|
|
|
@ -371,7 +371,7 @@ notification_object_data = {
|
|||
'BlockDevicePayload': '1.0-29751e1b6d41b1454e36768a1e764df8',
|
||||
'EventType': '1.10-2a6ab743d767837366ab1aec387d7c3b',
|
||||
'ExceptionNotification': '1.0-a73147b93b520ff0061865849d3dfa56',
|
||||
'ExceptionPayload': '1.0-27db46ee34cd97e39f2643ed92ad0cc5',
|
||||
'ExceptionPayload': '1.1-6c43008bd81885a63bc7f7c629f0793b',
|
||||
'FlavorNotification': '1.0-a73147b93b520ff0061865849d3dfa56',
|
||||
'FlavorPayload': '1.4-2e7011b8b4e59167fe8b7a0a81f0d452',
|
||||
'InstanceActionNotification': '1.0-a73147b93b520ff0061865849d3dfa56',
|
||||
|
|
|
@ -104,7 +104,7 @@ class WrapExceptionTestCase(test.NoDBTestCase):
|
|||
|
||||
payload = notification['payload']
|
||||
self.assertEqual('ExceptionPayload', payload['nova_object.name'])
|
||||
self.assertEqual('1.0', payload['nova_object.version'])
|
||||
self.assertEqual('1.1', payload['nova_object.version'])
|
||||
|
||||
payload = payload['nova_object.data']
|
||||
self.assertEqual('TestingException', payload['exception'])
|
||||
|
@ -112,6 +112,7 @@ class WrapExceptionTestCase(test.NoDBTestCase):
|
|||
self.assertEqual('bad_function_exception', payload['function_name'])
|
||||
self.assertEqual('nova.tests.unit.test_exception',
|
||||
payload['module_name'])
|
||||
self.assertIn('bad_function_exception', payload['traceback'])
|
||||
|
||||
@mock.patch('nova.rpc.NOTIFIER')
|
||||
@mock.patch('nova.notifications.objects.exception.'
|
||||
|
|
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
A new ``traceback`` field has been added to each versioned instance
|
||||
notification. In an error notification this field contains the full
|
||||
traceback string of the exception which caused the error notification.
|
||||
See the `notification dev ref`_ for the sample file of
|
||||
``instance.create.error`` as an example.
|
||||
|
||||
.. _notification dev ref: https://docs.openstack.org/nova/latest/reference/notifications.html#existing-versioned-notifications
|
Loading…
Reference in New Issue