Merge "Fix retry mechanism for generator results"

nova/image/glance.py

@@ -18,6 +18,7 @@
 from __future__ import absolute_import
 
 import copy
+import inspect
 import itertools
 import random
 import sys
@@ -246,7 +247,12 @@ class GlanceClientWrapper(object):
             client = self.client or self._create_onetime_client(context,
                                                                 version)
             try:
-                return getattr(client.images, method)(*args, **kwargs)
+                result = getattr(client.images, method)(*args, **kwargs)
+                if inspect.isgenerator(result):
+                    # Convert generator results to a list, so that we can
+                    # catch any potential exceptions now and retry the call.
+                    return list(result)
+                return result
             except retry_excs as e:
                 if attempt < num_attempts:
                     extra = "retrying"

nova/tests/unit/image/test_glance.py

@@ -381,6 +381,39 @@ class TestGlanceClientWrapper(test.NoDBTestCase):
         self.assertEqual(str(client.api_server), "https://host2:9293")
         sleep_mock.assert_called_once_with(1)
 
+    @mock.patch('random.shuffle')
+    @mock.patch('time.sleep')
+    @mock.patch('nova.image.glance._glanceclient_from_endpoint')
+    def test_retry_works_with_generators(self, create_client_mock,
+                                         sleep_mock, shuffle_mock):
+        def some_generator(exception):
+            if exception:
+                raise glanceclient.exc.CommunicationError('Boom!')
+            yield 'something'
+
+        api_servers = [
+            'https://host2:9292',
+            'https://host2:9293',
+            'http://host3:9294'
+        ]
+        client_mock = mock.MagicMock()
+        images_mock = mock.MagicMock()
+        images_mock.list.side_effect = [
+            some_generator(exception=True),
+            some_generator(exception=False),
+        ]
+        type(client_mock).images = mock.PropertyMock(return_value=images_mock)
+        create_client_mock.return_value = client_mock
+
+        self.flags(num_retries=1, group='glance')
+        self.flags(api_servers=api_servers, group='glance')
+
+        ctx = context.RequestContext('fake', 'fake')
+        client = glance.GlanceClientWrapper()
+        client.call(ctx, 1, 'list', 'meow')
+        sleep_mock.assert_called_once_with(1)
+        self.assertEqual(str(client.api_server), 'https://host2:9293')
+
     @mock.patch('oslo_service.sslutils.is_enabled')
     @mock.patch('glanceclient.Client')
     def test_create_glance_client_with_ssl(self, client_mock,