Fix race condition in instance.update sample test

The test case test_create_delete_server_with_instance_update failed randomly
because the last expected instance.update notification was not received due
to a race condition.

The test polls the REST API and waits for the instance to reach ACTIVE state.
However the last instance.update is sent from the instance.save after the
state of the instance is updated in the DB. Therefore it is possible that
the test saw ACTIVE instance state before the last instance.update
notification is emitted.

This patch introduce an extra wait call to receive the last notification
as well.

Closes-Bug: #1657087
Change-Id: I438bbb6de0733d0c3fd8361fdc4149c88abb53aa

nova/tests/functional/notification_sample_tests/notification_sample_base.py

@@ -199,3 +199,25 @@ class NotificationSampleTestBase(test.TestCase,
         self.assertTrue(
             received,
             'notification %s hasn\'t been received' % event_type)
+
+    def _wait_for_notifications(self, event_type, expected_count, timeout=1.0):
+        notifications = []
+        start_time = time.clock()
+
+        while (len(notifications) < expected_count
+               and time.clock() - start_time < timeout):
+
+            fake_notifier.wait_for_versioned_notification(event_type, timeout)
+            notifications += self._get_notifications(event_type)
+            # NOTE(gibi): reading and then resetting the fake_notifier without
+            # synchronization doesn't lead to race condition as the only
+            # parallelism is due to eventlet.
+            fake_notifier.reset()
+
+        self.assertEqual(expected_count, len(notifications),
+                         'Unexpected number of %s notifications '
+                         'within the given timeout. '
+                         'Expected %d, got %d: %s' %
+                         (event_type, expected_count, len(notifications),
+                          notifications))
+        return notifications

nova/tests/functional/notification_sample_tests/test_instance.py

@@ -153,7 +153,7 @@ class TestInstanceNotificationSample(
         server = self._boot_a_server(
             extra_params={'networks': [{'port': self.neutron.port_1['id']}]})
 
-        instance_updates = self._get_notifications('instance.update')
+        instance_updates = self._wait_for_notifications('instance.update', 7)
 
         # The first notification comes from the nova-api the rest is from the
         # nova-compute. To keep the test simpler assert this fact and then
@@ -163,10 +163,6 @@ class TestInstanceNotificationSample(
                          instance_updates[0]['publisher_id'])
         instance_updates[0]['publisher_id'] = 'nova-compute:fake-mini'
 
-        self.assertEqual(7, len(instance_updates),
-                         'Unexpected number of instance.update notifications. '
-                         'Expected 7, got %s: %s' % (
-                             len(instance_updates), instance_updates))
         create_steps = [
             # nothing -> scheduling
             {'reservation_id': server['reservation_id'],