diff --git a/marconi/tests/transport/wsgi/test_claims.py b/marconi/tests/transport/wsgi/test_claims.py index f3764444e..78069d0fa 100644 --- a/marconi/tests/transport/wsgi/test_claims.py +++ b/marconi/tests/transport/wsgi/test_claims.py @@ -29,6 +29,9 @@ class ClaimsBaseTest(base.TestBase): def setUp(self): super(ClaimsBaseTest, self).setUp() + self.wsgi_cfg = config.namespace( + 'drivers:transport:wsgi').from_options() + self.project_id = '480924' self.queue_path = '/v1/queues/fizbit' self.claims_path = self.queue_path + '/claims' @@ -64,6 +67,20 @@ class ClaimsBaseTest(base.TestBase): self.simulate_patch(href, self.project_id, body=doc) self.assertEquals(self.srmock.status, falcon.HTTP_400) + def test_too_much_metadata(self): + doc = '{"ttl": 100, "grace": 60}' + long_doc = doc + (' ' * + (self.wsgi_cfg.metadata_max_length - len(doc) + 1)) + + self.simulate_post(self.claims_path, self.project_id, body=long_doc) + self.assertEquals(self.srmock.status, falcon.HTTP_400) + + self.simulate_post(self.claims_path, self.project_id, body=doc) + href = self.srmock.headers_dict['Location'] + + self.simulate_patch(href, self.project_id, body=long_doc) + self.assertEquals(self.srmock.status, falcon.HTTP_400) + def test_lifecycle(self): doc = '{"ttl": 10, "grace": 30}' diff --git a/marconi/tests/transport/wsgi/test_messages.py b/marconi/tests/transport/wsgi/test_messages.py index 0dccd30a1..9ae4e45f0 100644 --- a/marconi/tests/transport/wsgi/test_messages.py +++ b/marconi/tests/transport/wsgi/test_messages.py @@ -18,6 +18,7 @@ import os import falcon +from marconi.common import config from marconi.tests.transport.wsgi import base @@ -26,6 +27,9 @@ class MessagesBaseTest(base.TestBase): def setUp(self): super(MessagesBaseTest, self).setUp() + self.wsgi_cfg = config.namespace( + 'drivers:transport:wsgi').from_options() + self.project_id = '7e55e1a7e' self.queue_path = '/v1/queues/fizbit' @@ -119,6 +123,18 @@ class MessagesBaseTest(base.TestBase): self.assertEquals(self.srmock.status, falcon.HTTP_400) + def test_exceeded_message_posting(self): + #TODO(zyuan): read `20` from the input validation module + doc = json.dumps([{'body': "some body", 'ttl': 100}] * 20, indent=4) + long_doc = doc + (' ' * + (self.wsgi_cfg.content_max_length - len(doc) + 1)) + + self.simulate_post(self.queue_path + '/messages', + body=long_doc, + headers=self.headers) + + self.assertEquals(self.srmock.status, falcon.HTTP_400) + def test_unsupported_json(self): for document in ('{"overflow": 9223372036854775808}', '{"underflow": -9223372036854775809}'): diff --git a/marconi/tests/transport/wsgi/test_queue_lifecycle.py b/marconi/tests/transport/wsgi/test_queue_lifecycle.py index df0f1b431..04b7a2856 100644 --- a/marconi/tests/transport/wsgi/test_queue_lifecycle.py +++ b/marconi/tests/transport/wsgi/test_queue_lifecycle.py @@ -22,13 +22,18 @@ import pymongo from marconi.common import config from marconi.tests.transport.wsgi import base -from marconi import transport class QueueLifecycleBaseTest(base.TestBase): config_filename = None + def setUp(self): + super(QueueLifecycleBaseTest, self).setUp() + + self.wsgi_cfg = config.namespace( + 'drivers:transport:wsgi').from_options() + def test_basics_thoroughly(self): path = '/v1/queues/gumshoe' @@ -95,7 +100,7 @@ class QueueLifecycleBaseTest(base.TestBase): self.simulate_put('/v1/queues/fizbat', '7e55e1a7e') self.assertEquals(self.srmock.status, falcon.HTTP_201) doc = '{"messages": {"ttl": 600}, "padding": "%s"}' - padding_len = transport.MAX_QUEUE_METADATA_SIZE - (len(doc) - 2) + 1 + padding_len = self.wsgi_cfg.metadata_max_length - (len(doc) - 2) + 1 doc = doc % ('x' * padding_len) self.simulate_put('/v1/queues/fizbat/metadata', '7e55e1a7e', body=doc) @@ -105,7 +110,7 @@ class QueueLifecycleBaseTest(base.TestBase): self.simulate_put('/v1/queues/fizbat', '7e55e1a7e') self.assertEquals(self.srmock.status, falcon.HTTP_201) doc = '{"messages": {"ttl": 600}, "padding": "%s"}' - padding_len = transport.MAX_QUEUE_METADATA_SIZE * 100 + padding_len = self.wsgi_cfg.metadata_max_length * 100 doc = doc % ('x' * padding_len) self.simulate_put('/v1/queues/fizbat/metadata', '7e55e1a7e', body=doc) @@ -117,7 +122,7 @@ class QueueLifecycleBaseTest(base.TestBase): # Set doc = '{"messages": {"ttl": 600}, "padding": "%s"}' - padding_len = transport.MAX_QUEUE_METADATA_SIZE - (len(doc) - 2) + padding_len = self.wsgi_cfg.metadata_max_length - (len(doc) - 2) doc = doc % ('x' * padding_len) self.simulate_put('/v1/queues/fizbat/metadata', '480924', body=doc) self.assertEquals(self.srmock.status, falcon.HTTP_204) diff --git a/marconi/transport/__init__.py b/marconi/transport/__init__.py index e7b3f9445..4b711ba54 100644 --- a/marconi/transport/__init__.py +++ b/marconi/transport/__init__.py @@ -9,9 +9,5 @@ OPTIONS = { CFG = config.project('marconi').from_options(**OPTIONS) -MAX_QUEUE_METADATA_SIZE = 64 * 1024 -"""Maximum metadata size per queue when serialized as JSON""" - - # Hoist into package namespace DriverBase = base.DriverBase diff --git a/marconi/transport/wsgi/claims.py b/marconi/transport/wsgi/claims.py index 96ca6d144..979072108 100644 --- a/marconi/transport/wsgi/claims.py +++ b/marconi/transport/wsgi/claims.py @@ -15,6 +15,7 @@ import falcon +from marconi.common import config import marconi.openstack.common.log as logging from marconi.storage import exceptions as storage_exceptions from marconi.transport import helpers @@ -23,6 +24,10 @@ from marconi.transport.wsgi import helpers as wsgi_helpers LOG = logging.getLogger(__name__) +CFG = config.namespace('drivers:transport:wsgi').from_options( + metadata_max_length=64 * 1024 +) + CLAIM_POST_SPEC = (('ttl', int), ('grace', int)) CLAIM_PATCH_SPEC = (('ttl', int),) @@ -38,10 +43,16 @@ class CollectionResource(object): LOG.debug(_("Claims collection POST - queue: %(queue)s, " "project: %(project)s") % {"queue": queue_name, "project": project_id}) + # Check for an explicit limit on the # of messages to claim limit = req.get_param_as_int('limit') claim_options = {} if limit is None else {'limit': limit} + # Place JSON size restriction before parsing + if req.content_length > CFG.metadata_max_length: + description = _('Claim metadata size is too large.') + raise wsgi_exceptions.HTTPBadRequestBody(description) + # Read claim metadata (e.g., TTL) and raise appropriate # HTTP errors as needed. metadata, = wsgi_helpers.filter_stream(req.stream, req.content_length, @@ -132,6 +143,12 @@ class ItemResource(object): {"queue_name": queue_name, "project_id": project_id, "claim_id": claim_id}) + + # Place JSON size restriction before parsing + if req.content_length > CFG.metadata_max_length: + description = _('Claim metadata size is too large.') + raise wsgi_exceptions.HTTPBadRequestBody(description) + # Read claim metadata (e.g., TTL) and raise appropriate # HTTP errors as needed. metadata, = wsgi_helpers.filter_stream(req.stream, req.content_length, diff --git a/marconi/transport/wsgi/messages.py b/marconi/transport/wsgi/messages.py index e188088bf..d08b18913 100644 --- a/marconi/transport/wsgi/messages.py +++ b/marconi/transport/wsgi/messages.py @@ -17,6 +17,7 @@ import itertools import falcon +from marconi.common import config import marconi.openstack.common.log as logging from marconi.storage import exceptions as storage_exceptions from marconi.transport import helpers @@ -25,6 +26,10 @@ from marconi.transport.wsgi import helpers as wsgi_helpers LOG = logging.getLogger(__name__) +CFG = config.namespace('drivers:transport:wsgi').from_options( + content_max_length=256 * 1024 +) + MESSAGE_POST_SPEC = (('ttl', int), ('body', '*')) @@ -135,6 +140,11 @@ class CollectionResource(object): uuid = req.get_header('Client-ID', required=True) + # Place JSON size restriction before parsing + if req.content_length > CFG.content_max_length: + description = _('Message collection size is too large.') + raise wsgi_exceptions.HTTPBadRequestBody(description) + # Pull out just the fields we care about messages = wsgi_helpers.filter_stream( req.stream, diff --git a/marconi/transport/wsgi/metadata.py b/marconi/transport/wsgi/metadata.py index cce285389..e1662b86d 100644 --- a/marconi/transport/wsgi/metadata.py +++ b/marconi/transport/wsgi/metadata.py @@ -15,14 +15,17 @@ import falcon +from marconi.common import config import marconi.openstack.common.log as logging from marconi.storage import exceptions as storage_exceptions -from marconi import transport from marconi.transport import helpers from marconi.transport.wsgi import exceptions as wsgi_exceptions LOG = logging.getLogger(__name__) +CFG = config.namespace('drivers:transport:wsgi').from_options( + metadata_max_length=64 * 1024 +) class Resource(object): @@ -57,8 +60,8 @@ class Resource(object): "project: %(project)s") % {"queue": queue_name, "project": project_id}) - # TODO(kgriffs): Migrate this check to input validator middleware - if req.content_length > transport.MAX_QUEUE_METADATA_SIZE: + # Place JSON size restriction before parsing + if req.content_length > CFG.metadata_max_length: description = _('Queue metadata size is too large.') raise wsgi_exceptions.HTTPBadRequestBody(description)