openstack
/
nova
Code Issues Proposed changes

Handle InstanceNotFound when sending instance update notification 

When Tempest (or any automated client) is waiting for an instance to be
ACTIVE but goes to ERROR state, it immediately fails a test and deletes
the instance.  The instance could be deleted by the time the instance
update notification processing happens so there is a race where that
could fail with InstanceNotFound.  That will get traced to the conductor
logs and we should try to keep the conductor logs as clean as possible
from expected exceptions.

This handles InstanceNotFound and just logs it at debug rather than
trace out the exception.

Closes-Bug: #1455640

Change-Id: Iaee147051afb59f7991f8dfef3bd3072c28f829d
This commit is contained in:
Matt Riedemann 2015-05-15 16:01:48 -07:00
parent 879f060e7a
commit 7de1235966
2 changed files with 39 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
nova/notifications.py
View File

@ -28,6 +28,7 @@ from oslo_utils import timeutils
import six
import nova.context
from nova import exception
from nova.i18n import _LE
from nova.image import glance
from nova import network
@ -150,6 +151,10 @@ def send_update(context, old_instance, new_instance, service="compute",
_send_instance_update_notification(context, new_instance,
service=service, host=host,
old_display_name=old_display_name)
except exception.InstanceNotFound:
LOG.debug('Failed to send instance update notification. The '
'instance could not be found and was most likely '
'deleted.', instance=new_instance)
except Exception:
LOG.exception(_LE("Failed to send state update notification"),
instance=new_instance)
@ -189,6 +194,10 @@ def send_update_with_states(context, instance, old_vm_state, new_vm_state,
old_vm_state=old_vm_state, old_task_state=old_task_state,
new_vm_state=new_vm_state, new_task_state=new_task_state,
service=service, host=host)
except exception.InstanceNotFound:
LOG.debug('Failed to send instance update notification. The '
'instance could not be found and was most likely '
'deleted.', instance=instance)
except Exception:
LOG.exception(_LE("Failed to send state update notification"),
instance=instance)

30
nova/tests/unit/test_notifications.py
View File

@ -26,6 +26,7 @@ from nova.compute import flavors
from nova.compute import task_states
from nova.compute import vm_states
from nova import context
from nova import exception
from nova.network import api as network_api
from nova import notifications
from nova import objects
@ -432,6 +433,35 @@ class NotificationsTestCase(test.TestCase):
notifications.send_update(self.context, self.instance, self.instance)
self.assertEqual(0, len(fake_notifier.NOTIFICATIONS))
@mock.patch.object(notifications.LOG, 'exception')
def test_fail_sending_update_instance_not_found(self, mock_log_exception):
# Tests that InstanceNotFound is handled as an expected exception and
# not logged as an error.
notfound = exception.InstanceNotFound(instance_id=self.instance.uuid)
with mock.patch.object(notifications,
'_send_instance_update_notification',
side_effect=notfound):
notifications.send_update(
self.context, self.instance, self.instance)
self.assertEqual(0, len(fake_notifier.NOTIFICATIONS))
self.assertEqual(0, mock_log_exception.call_count)
@mock.patch.object(notifications.LOG, 'exception')
def test_fail_send_update_with_states_inst_not_found(self,
mock_log_exception):
# Tests that InstanceNotFound is handled as an expected exception and
# not logged as an error.
notfound = exception.InstanceNotFound(instance_id=self.instance.uuid)
with mock.patch.object(notifications,
'_send_instance_update_notification',
side_effect=notfound):
notifications.send_update_with_states(
self.context, self.instance,
vm_states.BUILDING, vm_states.ERROR,
task_states.NETWORKING, new_task_state=None)
self.assertEqual(0, len(fake_notifier.NOTIFICATIONS))
self.assertEqual(0, mock_log_exception.call_count)
class NotificationsFormatTestCase(test.NoDBTestCase):