feat(storage): configurable default paging size

This change add the following options to the config file:

    [limits:storage]
    default_queue_paging = 10
    default_message_paging = 10

So that the default value of the  "limit" URI param is now configurable.

This patch also removes the "actions" cruft.

Implements: blueprint configurable-default-paging
Change-Id: Id38295f1e607226a4259be7744e6ce2d7b6de12e

etc/marconi.conf-sample

@@ -84,3 +84,9 @@ database = marconi
 # for each metadata body and each message body
 ;metadata_size_uplimit = 65536
 ;message_size_uplimit = 262144
+
+[limits:storage]
+# The default number of queue records per page when listing queues
+;default_queue_paging = 10
+# The default number of messages per page when listing or claiming messages
+;default_message_paging = 10

marconi/storage/base.py

@@ -90,7 +90,8 @@ class QueueBase(ControllerBase):
 
         :param project: Project id
         :param marker: The last queue name
-        :param limit: (Default 10) Max number
+        :param limit: (Default 10, configurable) Max number
+            queues to return.
         :param detailed: Whether metadata is included
         :param include_claimed: Whether to list claimed messages
 
@@ -164,18 +165,6 @@ class QueueBase(ControllerBase):
         """
         raise NotImplementedError
 
-    @abc.abstractmethod
-    def actions(self, name, project=None, marker=None, limit=10):
-        """Base method for queue actions.
-
-        :param name: Queue name
-        :param project: Project id
-        :param marker: Tail identifier
-        :param limit: (Default 10) Max number
-            of messages to retrieve.
-        """
-        raise NotImplementedError
-
 
 class MessageBase(ControllerBase):
     """This class is responsible for managing message CRUD."""
@@ -191,7 +180,7 @@ class MessageBase(ControllerBase):
             message from.
         :param project: Project id
         :param marker: Tail identifier
-        :param limit: (Default 10) specifies up to 100
+        :param limit: (Default 10, configurable) Max number
             messages to return.
         :param echo: (Default False) Boolean expressing whether
             or not this client should receive its own messages.
@@ -302,7 +291,7 @@ class ClaimBase(ControllerBase):
         :param metadata: Claim's parameters
             to be stored.
         :param project: Project id
-        :param limit: (Default 10) Max number
+        :param limit: (Default 10, configurable) Max number
             of messages to claim.
 
         :returns: (Claim ID, claimed messages)

marconi/storage/mongodb/claims.py

@@ -25,14 +25,17 @@ import datetime
 
 from bson import objectid
 
+from marconi.common import config
 import marconi.openstack.common.log as logging
 from marconi.openstack.common import timeutils
 from marconi import storage
 from marconi.storage import exceptions
 from marconi.storage.mongodb import utils
 
-
 LOG = logging.getLogger(__name__)
+CFG = config.namespace('limits:storage').from_options(
+    default_message_paging=10,
+)
 
 
 class ClaimController(storage.ClaimBase):
@@ -98,7 +101,7 @@ class ClaimController(storage.ClaimBase):
         return (claim, msgs)
 
     @utils.raises_conn_error
-    def create(self, queue, metadata, project=None, limit=10):
+    def create(self, queue, metadata, project=None, limit=None):
         """Creates a claim.
 
         This implementation was done in a best-effort fashion.
@@ -118,6 +121,9 @@ class ClaimController(storage.ClaimBase):
         """
         msg_ctrl = self.driver.message_controller
 
+        if limit is None:
+            limit = CFG.default_message_paging
+
         ttl = metadata['ttl']
         grace = metadata['grace']
         oid = objectid.ObjectId()

marconi/storage/mongodb/messages.py

@@ -26,6 +26,7 @@ import time
 
 import pymongo.errors
 
+from marconi.common import config
 import marconi.openstack.common.log as logging
 from marconi.openstack.common import timeutils
 from marconi import storage
@@ -34,6 +35,9 @@ from marconi.storage.mongodb import options
 from marconi.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
+CFG = config.namespace('limits:storage').from_options(
+    default_message_paging=10,
+)
 
 
 class MessageController(storage.MessageBase):
@@ -397,9 +401,12 @@ class MessageController(storage.MessageBase):
         for name, project in self._queue_controller._get_np():
             self._remove_expired(name, project)
 
-    def list(self, queue_name, project=None, marker=None, limit=10,
+    def list(self, queue_name, project=None, marker=None, limit=None,
              echo=False, client_uuid=None, include_claimed=False):
 
+        if limit is None:
+            limit = CFG.default_message_paging
+
         if marker is not None:
             try:
                 marker = int(marker)

marconi/storage/mongodb/queues.py

@@ -23,6 +23,7 @@ Field Mappings:
 
 import pymongo.errors
 
+from marconi.common import config
 import marconi.openstack.common.log as logging
 from marconi.openstack.common import timeutils
 from marconi import storage
@@ -30,6 +31,9 @@ from marconi.storage import exceptions
 from marconi.storage.mongodb import utils
 
 LOG = logging.getLogger(__name__)
+CFG = config.namespace('limits:storage').from_options(
+    default_queue_paging=10,
+)
 
 
 class QueueController(storage.QueueBase):
@@ -77,7 +81,11 @@ class QueueController(storage.QueueBase):
     #-----------------------------------------------------------------------
 
     def list(self, project=None, marker=None,
-             limit=10, detailed=False):
+             limit=None, detailed=False):
+
+        if limit is None:
+            limit = CFG.default_queue_paging
+
         query = {'p': project}
         if marker:
             query['n'] = {'$gt': marker}
@@ -161,7 +169,3 @@ class QueueController(storage.QueueBase):
             message_stats['newest'] = utils.stat_message(newest, now)
 
         return {'messages': message_stats}
-
-    @utils.raises_conn_error
-    def actions(self, name, project=None, marker=None, limit=10):
-        raise NotImplementedError

marconi/storage/sqlite/claims.py

@@ -13,10 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from marconi.common import config
 from marconi.storage import base
 from marconi.storage import exceptions
 from marconi.storage.sqlite import utils
 
+CFG = config.namespace('limits:storage').from_options(
+    default_message_paging=10,
+)
+
 
 class ClaimController(base.ClaimBase):
     def __init__(self, driver):
@@ -73,10 +78,14 @@ class ClaimController(base.ClaimBase):
             except utils.NoResult:
                 raise exceptions.ClaimDoesNotExist(claim_id, queue, project)
 
-    def create(self, queue, metadata, project, limit=10):
+    def create(self, queue, metadata, project, limit=None):
+
         if project is None:
             project = ''
 
+        if limit is None:
+            limit = CFG.default_message_paging
+
         with self.driver('immediate'):
             try:
                 qid = utils.get_qid(self.driver, queue, project)

marconi/storage/sqlite/messages.py

@@ -13,11 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from marconi.common import config
 from marconi.openstack.common import timeutils
 from marconi.storage import base
 from marconi.storage import exceptions
 from marconi.storage.sqlite import utils
 
+CFG = config.namespace('limits:storage').from_options(
+    default_message_paging=10,
+)
+
 
 class MessageController(base.MessageBase):
     def __init__(self, driver):
@@ -130,9 +135,12 @@ class MessageController(base.MessageBase):
                 'body': content,
             }
 
-    def list(self, queue, project, marker=None, limit=10,
+    def list(self, queue, project, marker=None, limit=None,
              echo=False, client_uuid=None, include_claimed=False):
 
+        if limit is None:
+            limit = CFG.default_message_paging
+
         if project is None:
             project = ''
 

marconi/storage/sqlite/queues.py

@@ -14,10 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from marconi.common import config
 from marconi.storage import base
 from marconi.storage import exceptions
 from marconi.storage.sqlite import utils
 
+CFG = config.namespace('limits:storage').from_options(
+    default_queue_paging=10,
+)
+
 
 class QueueController(base.QueueBase):
     def __init__(self, driver):
@@ -36,11 +41,14 @@ class QueueController(base.QueueBase):
         ''')
 
     def list(self, project, marker=None,
-             limit=10, detailed=False):
+             limit=None, detailed=False):
 
         if project is None:
             project = ''
 
+        if limit is None:
+            limit = CFG.default_queue_paging
+
         sql = (('''
             select name from Queues''' if not detailed
                 else '''
@@ -166,6 +174,3 @@ class QueueController(base.QueueBase):
                 message_stats['newest'] = utils.stat_message(newest)
 
             return {'messages': message_stats}
-
-    def actions(self, name, project, marker=None, limit=10):
-        raise NotImplementedError

marconi/tests/etc/wsgi_sqlite_default_limits.conf

@@ -0,0 +1,7 @@
+[drivers]
+transport = wsgi
+storage = sqlite
+
+[limits:storage]
+default_queue_paging = 1
+default_message_paging = 2

marconi/tests/transport/wsgi/test_default_limits.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2013 Rackspace, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+import falcon
+
+from marconi.tests.transport.wsgi import base
+
+
+class DefaultLimitsTest(base.TestBase):
+
+    config_filename = 'wsgi_sqlite_default_limits.conf'
+
+    def setUp(self):
+        super(DefaultLimitsTest, self).setUp()
+
+        self.queue_path = '/v1/queues/q1'
+        self.messages_path = self.queue_path + '/messages'
+        self.claims_path = self.queue_path + '/claims'
+
+        self.simulate_put(self.queue_path)
+
+    def tearDown(self):
+        self.simulate_delete(self.queue_path)
+        super(DefaultLimitsTest, self).tearDown()
+
+    def test_queue_listing(self):
+        default_queue_paging = 1
+
+        # 2 queues to list
+        self.simulate_put('/v1/queues/q2')
+        self.assertEquals(self.srmock.status, falcon.HTTP_201)
+
+        result = self.simulate_get('/v1/queues')
+        self.assertEquals(self.srmock.status, falcon.HTTP_200)
+
+        queues = json.loads(result[0])['queues']
+        self.assertEquals(len(queues), default_queue_paging)
+
+        self.simulate_delete('/v1/queues/q2')
+
+    def test_message_listing(self):
+        default_message_paging = 2
+
+        # 10 messages to list
+        self.__prepare_messages(10)
+
+        result = self.simulate_get(self.messages_path,
+                                   headers={'Client-ID': 'audience'})
+
+        self.assertEquals(self.srmock.status, falcon.HTTP_200)
+
+        messages = json.loads(result[0])['messages']
+        self.assertEquals(len(messages), default_message_paging)
+
+    def test_claim_creation(self):
+        default_message_paging = 2
+
+        # 5 messages to claim
+        self.__prepare_messages(5)
+
+        result = self.simulate_post(self.claims_path,
+                                    body='{"ttl": 60, "grace": 60}')
+
+        self.assertEquals(self.srmock.status, falcon.HTTP_201)
+
+        messages = json.loads(result[0])
+        self.assertEquals(len(messages), default_message_paging)
+
+    def __prepare_messages(self, count):
+        doc = json.dumps([{'body': 239, 'ttl': 300}] * count)
+        self.simulate_post(self.messages_path, body=doc,
+                           headers={'Client-ID': 'poster'})

marconi/tests/util/faulty_storage.py

@@ -56,9 +56,6 @@ class QueueController(storage.QueueBase):
     def stats(self, name, project=None):
         raise NotImplementedError()
 
-    def actions(self, name, project=None, marker=None, limit=10):
-        raise NotImplementedError()
-
 
 class MessageController(storage.MessageBase):
     def __init__(self, driver):