Make import wait check for store status

During a glance import, if we fail, we are not going to change the
state of the image in all cases, and definitely not to 'active'.
Thus waiting for active just means we keep polling a broken image
for a long time, wasting resources. We should also be checking the
list of failed stores, and if something pops in there, abort right
then and there.

This patch makes us use the wait_for_image_imported_to_stores()
waiter, and modifies it to not look at image['stores'] if no
stores are provided. In the case where we don't have multistore
support enabled, we won't be able to know that the import failed
until we time out, but otherwise we will (barring a glance bug for
which I also have a fix). This also makes the waiter not fail on
KeyError if os_glance_failed_import is not present on the image,
as would be the case if used when stores are not enabled.

Note that I'm doing this because sometimes web-download fails in the
gate due to a timeout trying to pull our http_image and we just keep
looping until our own timeout instead of noticing. It also means
we just report "never reached active state" instead of what we know
to be true, which is that import failed.

Change-Id: Ie0569b84dd1c397a79d13af609bf23db52a2dfc8

tempest/api/image/v2/test_images.py

@@ -90,7 +90,7 @@ class ImportImagesTest(base.BaseV2ImageTest):
         self.assertEqual('uploading', body['status'])
         # import image from staging to backend
         self.client.image_import(image['id'], method='glance-direct')
-        self.client.wait_for_resource_activation(image['id'])
+        waiters.wait_for_image_imported_to_stores(self.client, image['id'])
 
     @decorators.idempotent_id('f6feb7a4-b04f-4706-a011-206129f83e62')
     def test_image_web_download_import(self):
@@ -111,7 +111,7 @@ class ImportImagesTest(base.BaseV2ImageTest):
         image_uri = CONF.image.http_image
         self.client.image_import(image['id'], method='web-download',
                                  image_uri=image_uri)
-        self.client.wait_for_resource_activation(image['id'])
+        waiters.wait_for_image_imported_to_stores(self.client, image['id'])
 
 
 class MultiStoresImportImagesTest(base.BaseV2ImageTest):

tempest/common/waiters.py

@@ -193,26 +193,34 @@ def wait_for_image_status(client, image_id, status):
     raise lib_exc.TimeoutException(message)
 
 
-def wait_for_image_imported_to_stores(client, image_id, stores):
+def wait_for_image_imported_to_stores(client, image_id, stores=None):
     """Waits for an image to be imported to all requested stores.
 
+    Short circuits to fail if the serer reports failure of any store.
+    If stores is None, just wait for status==active.
+
     The client should also have build_interval and build_timeout attributes.
     """
 
+    exc_cls = lib_exc.TimeoutException
     start = int(time.time())
     while int(time.time()) - start < client.build_timeout:
         image = client.show_image(image_id)
-        if image['status'] == 'active' and image['stores'] == stores:
+        if image['status'] == 'active' and (stores is None or
+                                            image['stores'] == stores):
             return
+        if image.get('os_glance_failed_import'):
+            exc_cls = lib_exc.OtherRestClientException
+            break
 
         time.sleep(client.build_interval)
 
     message = ('Image %s failed to import on stores: %s' %
-               (image_id, str(image['os_glance_failed_import'])))
+               (image_id, str(image.get('os_glance_failed_import'))))
     caller = test_utils.find_test_caller()
     if caller:
         message = '(%s) %s' % (caller, message)
-    raise lib_exc.TimeoutException(message)
+    raise exc_cls(message)
 
 
 def wait_for_image_copied_to_stores(client, image_id):

tempest/tests/common/test_waiters.py

@@ -66,7 +66,7 @@ class TestImageWaiters(base.TestCase):
         # Ensure waiter returns before build_timeout
         self.assertLess((end_time - start_time), 10)
 
-    def test_wait_for_image_imported_to_stores_timeout(self):
+    def test_wait_for_image_imported_to_stores_failure(self):
         time_mock = self.patch('time.time')
         client = mock.MagicMock()
         client.build_timeout = 2
@@ -77,6 +77,20 @@ class TestImageWaiters(base.TestCase):
             'status': 'saving',
             'stores': 'fake_store',
             'os_glance_failed_import': 'fake_os_glance_failed_import'})
+        self.assertRaises(lib_exc.OtherRestClientException,
+                          waiters.wait_for_image_imported_to_stores,
+                          client, 'fake_image_id', 'fake_store')
+
+    def test_wait_for_image_imported_to_stores_timeout(self):
+        time_mock = self.patch('time.time')
+        client = mock.MagicMock()
+        client.build_timeout = 2
+        self.patch('time.time', side_effect=[0., 1., 2.])
+        time_mock.side_effect = utils.generate_timeout_series(1)
+
+        client.show_image.return_value = ({
+            'status': 'saving',
+            'stores': 'fake_store'})
         self.assertRaises(lib_exc.TimeoutException,
                           waiters.wait_for_image_imported_to_stores,
                           client, 'fake_image_id', 'fake_store')