Catch NoPoolFound on resource listing in transport

For example, "queue_list" operation in Websocket transport should catch
any storage errors to return 503 error response, but the line
"queues = list(next(results))", which can throw NoPoolFound exception,
is outside try block.

This problem not only affects "queue_list" operation in Websocket
transport, but also some other places.

This patch moves such lines inside try blocks, so exception will be
properly handled. This patch also adds tests for this specific case.

Change-Id: I5557aee5c0e4c1127f54b2059164bd1a0627c89d
Closes-Bug: 1538795

zaqar/api/v2/endpoints.py

@@ -62,6 +62,8 @@ class Endpoints(object):
             self._validate.queue_listing(**kwargs)
             results = self._queue_controller.list(
                 project=project_id, **kwargs)
+            # Buffer list of queues. Can raise NoPoolFound error.
+            queues = list(next(results))
         except (ValueError, validation.ValidationFailed) as ex:
             LOG.debug(ex)
             headers = {'status': 400}
@@ -72,9 +74,6 @@ class Endpoints(object):
             headers = {'status': 503}
             return api_utils.error_response(req, ex, headers, error)
 
-        # Buffer list of queues
-        queues = list(next(results))
-
         # Got some. Prepare the response.
         body = {'queues': queues}
         headers = {'status': 200}
@@ -750,6 +749,8 @@ class Endpoints(object):
             self._validate.subscription_listing(**kwargs)
             results = self._subscription_controller.list(
                 queue_name, project=project_id, **kwargs)
+            # Buffer list of subscriptions. Can raise NoPoolFound error.
+            subscriptions = list(next(results))
         except (ValueError, validation.ValidationFailed) as ex:
             LOG.debug(ex)
             headers = {'status': 400}
@@ -760,9 +761,6 @@ class Endpoints(object):
             headers = {'status': 503}
             return api_utils.error_response(req, ex, headers, error)
 
-        # Buffer list of queues
-        subscriptions = list(next(results))
-
         # Got some. Prepare the response.
         body = {'subscriptions': subscriptions}
         headers = {'status': 200}

zaqar/tests/unit/transport/websocket/v2/test_queue_lifecycle.py

@@ -18,6 +18,7 @@ import uuid
 import ddt
 import mock
 
+from zaqar.storage import errors as storage_errors
 from zaqar import tests as testing
 from zaqar.tests.unit.transport.websocket import base
 from zaqar.tests.unit.transport.websocket import utils as test_utils
@@ -566,6 +567,41 @@ class QueueLifecycleBaseTest(base.V2Base):
         req = test_utils.create_request(action, body, headers)
         self.protocol.onMessage(req, False)
 
+    def test_list_returns_503_on_nopoolfound_exception(self):
+        headers = {
+            'Client-ID': str(uuid.uuid4()),
+            'X-Project-ID': 'test-project'
+        }
+        action = "queue_list"
+        body = {}
+
+        send_mock = mock.patch.object(self.protocol, 'sendMessage')
+        self.addCleanup(send_mock.stop)
+        sender = send_mock.start()
+
+        req = test_utils.create_request(action, body, headers)
+
+        def validator(resp, isBinary):
+            resp = json.loads(resp)
+            self.assertEqual(503, resp['headers']['status'])
+
+        sender.side_effect = validator
+
+        queue_controller = self.boot.storage.queue_controller
+
+        with mock.patch.object(queue_controller, 'list') as mock_queue_list:
+
+            def queue_generator():
+                raise storage_errors.NoPoolFound()
+
+            # This generator tries to be like queue controller list generator
+            # in some ways.
+            def fake_generator():
+                yield queue_generator()
+                yield {}
+            mock_queue_list.return_value = fake_generator()
+            self.protocol.onMessage(req, False)
+
 
 class TestQueueLifecycleMongoDB(QueueLifecycleBaseTest):
 

zaqar/tests/unit/transport/websocket/v2/test_subscriptions.py

@@ -18,6 +18,7 @@ import uuid
 
 import mock
 
+from zaqar.storage import errors as storage_errors
 from zaqar.tests.unit.transport.websocket import base
 from zaqar.tests.unit.transport.websocket import utils as test_utils
 from zaqar.transport.websocket import factory
@@ -220,3 +221,40 @@ class SubscriptionTest(base.V1_1Base):
 
         self.assertEqual(1, sender.call_count)
         self.assertEqual(response, json.loads(sender.call_args[0][0]))
+
+    def test_list_returns_503_on_nopoolfound_exception(self):
+        sub = self.boot.storage.subscription_controller.create(
+            'kitkat', '', 600, {}, project=self.project_id)
+        self.addCleanup(
+            self.boot.storage.subscription_controller.delete, 'kitkat', sub,
+            project=self.project_id)
+        action = 'subscription_list'
+        body = {'queue_name': 'kitkat'}
+
+        send_mock = mock.patch.object(self.protocol, 'sendMessage')
+        self.addCleanup(send_mock.stop)
+        sender = send_mock.start()
+
+        req = test_utils.create_request(action, body, self.headers)
+
+        def validator(resp, isBinary):
+            resp = json.loads(resp)
+            self.assertEqual(503, resp['headers']['status'])
+
+        sender.side_effect = validator
+
+        subscription_controller = self.boot.storage.subscription_controller
+
+        with mock.patch.object(subscription_controller, 'list') as \
+                mock_subscription_list:
+
+            def subscription_generator():
+                raise storage_errors.NoPoolFound()
+
+            # This generator tries to be like subscription controller list
+            # generator in some ways.
+            def fake_generator():
+                yield subscription_generator()
+                yield {}
+            mock_subscription_list.return_value = fake_generator()
+            self.protocol.onMessage(req, False)

zaqar/tests/unit/transport/wsgi/v1/test_queue_lifecycle.py

@@ -14,9 +14,11 @@
 
 import ddt
 import falcon
+import mock
 from oslo_serialization import jsonutils
 import six
 
+from zaqar.storage import errors as storage_errors
 from zaqar import tests as testing
 from zaqar.tests.unit.transport.wsgi import base
 
@@ -347,6 +349,29 @@ class TestQueueLifecycleMongoDB(base.V1Base):
         self.simulate_get(target, project_id, query_string='marker=zzz')
         self.assertEqual(falcon.HTTP_204, self.srmock.status)
 
+    def test_list_returns_503_on_nopoolfound_exception(self):
+        arbitrary_number = 644079696574693
+        project_id = str(arbitrary_number)
+        header = {
+            'X-Project-ID': project_id,
+        }
+
+        queue_controller = self.boot.storage.queue_controller
+
+        with mock.patch.object(queue_controller, 'list') as mock_queue_list:
+
+            def queue_generator():
+                raise storage_errors.NoPoolFound()
+
+            # This generator tries to be like queue controller list generator
+            # in some ways.
+            def fake_generator():
+                yield queue_generator()
+                yield {}
+            mock_queue_list.return_value = fake_generator()
+            self.simulate_get(self.queue_path, headers=header)
+            self.assertEqual(falcon.HTTP_503, self.srmock.status)
+
 
 class TestQueueLifecycleFaultyDriver(base.V1BaseFaulty):
 

zaqar/tests/unit/transport/wsgi/v1_1/test_queue_lifecycle.py

@@ -16,9 +16,11 @@ import uuid
 
 import ddt
 import falcon
+import mock
 from oslo_serialization import jsonutils
 import six
 
+from zaqar.storage import errors as storage_errors
 from zaqar import tests as testing
 from zaqar.tests.unit.transport.wsgi import base
 
@@ -325,6 +327,31 @@ class TestQueueLifecycleMongoDB(base.V1_1Base):
         self.simulate_get(target, headers=header, query_string='marker=zzz')
         self.assertEqual(falcon.HTTP_200, self.srmock.status)
 
+    def test_list_returns_503_on_nopoolfound_exception(self):
+        arbitrary_number = 644079696574693
+        project_id = str(arbitrary_number)
+        client_id = str(uuid.uuid4())
+        header = {
+            'X-Project-ID': project_id,
+            'Client-ID': client_id
+        }
+
+        queue_controller = self.boot.storage.queue_controller
+
+        with mock.patch.object(queue_controller, 'list') as mock_queue_list:
+
+            def queue_generator():
+                raise storage_errors.NoPoolFound()
+
+            # This generator tries to be like queue controller list generator
+            # in some ways.
+            def fake_generator():
+                yield queue_generator()
+                yield {}
+            mock_queue_list.return_value = fake_generator()
+            self.simulate_get(self.queue_path, headers=header)
+            self.assertEqual(falcon.HTTP_503, self.srmock.status)
+
 
 class TestQueueLifecycleFaultyDriver(base.V1_1BaseFaulty):
 

zaqar/tests/unit/transport/wsgi/v2_0/test_queue_lifecycle.py

@@ -16,9 +16,11 @@ import uuid
 
 import ddt
 import falcon
+import mock
 from oslo_serialization import jsonutils
 import six
 
+from zaqar.storage import errors as storage_errors
 from zaqar import tests as testing
 from zaqar.tests.unit.transport.wsgi import base
 
@@ -326,6 +328,31 @@ class TestQueueLifecycleMongoDB(base.V2Base):
         self.simulate_get(target, headers=header, query_string='marker=zzz')
         self.assertEqual(falcon.HTTP_200, self.srmock.status)
 
+    def test_list_returns_503_on_nopoolfound_exception(self):
+        arbitrary_number = 644079696574693
+        project_id = str(arbitrary_number)
+        client_id = str(uuid.uuid4())
+        header = {
+            'X-Project-ID': project_id,
+            'Client-ID': client_id
+        }
+
+        queue_controller = self.boot.storage.queue_controller
+
+        with mock.patch.object(queue_controller, 'list') as mock_queue_list:
+
+            def queue_generator():
+                raise storage_errors.NoPoolFound()
+
+            # This generator tries to be like queue controller list generator
+            # in some ways.
+            def fake_generator():
+                yield queue_generator()
+                yield {}
+            mock_queue_list.return_value = fake_generator()
+            self.simulate_get(self.queue_path, headers=header)
+            self.assertEqual(falcon.HTTP_503, self.srmock.status)
+
 
 class TestQueueLifecycleFaultyDriver(base.V2BaseFaulty):
 

zaqar/tests/unit/transport/wsgi/v2_0/test_subscriptions.py

@@ -16,8 +16,10 @@ import uuid
 
 import ddt
 import falcon
+import mock
 from oslo_serialization import jsonutils
 
+from zaqar.storage import errors as storage_errors
 from zaqar import tests as testing
 from zaqar.tests.unit.transport.wsgi import base
 
@@ -176,6 +178,32 @@ class TestSubscriptionsMongoDB(base.V2Base):
     def test_list_works(self):
         self._list_subscription()
 
+    def test_list_returns_503_on_nopoolfound_exception(self):
+        arbitrary_number = 644079696574693
+        project_id = str(arbitrary_number)
+        client_id = str(uuid.uuid4())
+        header = {
+            'X-Project-ID': project_id,
+            'Client-ID': client_id
+        }
+
+        subscription_controller = self.boot.storage.subscription_controller
+
+        with mock.patch.object(subscription_controller, 'list') as \
+                mock_subscription_list:
+
+            def subscription_generator():
+                raise storage_errors.NoPoolFound()
+
+            # This generator tries to be like subscription controller list
+            # generator in some ways.
+            def fake_generator():
+                yield subscription_generator()
+                yield {}
+            mock_subscription_list.return_value = fake_generator()
+            self.simulate_get(self.subscription_path, headers=header)
+            self.assertEqual(falcon.HTTP_503, self.srmock.status)
+
     def test_list_empty(self):
         resp = self.simulate_get(self.subscription_path,
                                  headers=self.headers)

zaqar/transport/wsgi/v2_0/subscriptions.py

@@ -126,6 +126,8 @@ class CollectionResource(object):
             results = self._subscription_controller.list(queue_name,
                                                          project=project_id,
                                                          **kwargs)
+            # Buffer list of subscriptions. Can raise NoPoolFound error.
+            subscriptions = list(next(results))
         except validation.ValidationFailed as ex:
             LOG.debug(ex)
             raise wsgi_errors.HTTPBadRequestAPI(six.text_type(ex))
@@ -135,9 +137,6 @@ class CollectionResource(object):
             description = _(u'Subscriptions could not be listed.')
             raise wsgi_errors.HTTPServiceUnavailable(description)
 
-        # Buffer list of subscriptions
-        subscriptions = list(next(results))
-
         # Got some. Prepare the response.
         kwargs['marker'] = next(results) or kwargs.get('marker', '')