openstack
/
zaqar
Code Issues Proposed changes

Support delayed queues for mongo 

DocImpact
ApiImpact

Tempest plugin
Depends-On: I0a15600a836f609e4b992f8a80ba887e312aa780

Implement blueprint delayed-queues
Change-Id: I7f29ef75318017a4fc1468ec57890ffcba96b923
This commit is contained in:
yangzhenyu 2017-10-20 10:35:12 +08:00 committed by gecong
parent c54318727b
commit 4fe01ea83b
10 changed files with 108 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
zaqar/common/api/utils.py
View File

@ -172,6 +172,10 @@ def get_headers(req):
kwargs['include_claimed'] = strutils.bool_from_string(
req._body.get('include_claimed'))
if req._body.get('include_delayed') is not None:
kwargs['include_delayed'] = strutils.bool_from_string(
req._body.get('include_delayed'))
if req._body.get('ttl') is not None:
kwargs['ttl'] = int(req._body.get('ttl'))

4
zaqar/storage/base.py
View File

@ -414,7 +414,7 @@ class Message(ControllerBase):
def list(self, queue, project=None, marker=None,
limit=DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None,
include_claimed=False):
include_claimed=False, include_delayed=False):
"""Base method for listing messages.
:param queue: Name of the queue to get the
@ -428,6 +428,8 @@ class Message(ControllerBase):
:param client_uuid: A UUID object. Required when echo=False.
:param include_claimed: omit claimed messages from listing?
:type include_claimed: bool
:param include_delayed: omit delayed messages from listing
:type include_delayed: bool
:returns: An iterator giving a sequence of messages and
the marker of the next page.

15
zaqar/storage/mongodb/claims.py
View File

@ -130,7 +130,9 @@ class ClaimController(storage.Claim):
"""
msg_ctrl = self.driver.message_controller
queue_ctrl = self.driver.queue_controller
# Get the maxClaimCount, deadLetterQueue and DelayTTL
# from current queue's meta
queue_meta = queue_ctrl.get(queue, project=project)
ttl = metadata['ttl']
grace = metadata['grace']
oid = objectid.ObjectId()
@ -150,11 +152,17 @@ class ClaimController(storage.Claim):
'c': 0 # NOTE(flwang): A placeholder which will be updated later
}
# NOTE(cdyangzhenyu): If the ``_default_message_delay`` is 0 means
# queue is not delayed queue, So we don't filter for delay messages.
include_delayed = False if queue_meta.get('_default_message_delay',
0) else True
# Get a list of active, not claimed nor expired
# messages that could be claimed.
msgs = msg_ctrl._active(queue, projection={'_id': 1, 'c': 1},
project=project,
limit=limit)
limit=limit,
include_delayed=include_delayed)
messages = iter([])
be_claimed = [(msg['_id'], msg['c'].get('c', 0)) for msg in msgs]
@ -163,9 +171,6 @@ class ClaimController(storage.Claim):
if len(ids) == 0:
return None, messages
# Get the maxClaimCount and deadLetterQueue from current queue's meta
queue_meta = queue_ctrl.get(queue, project=project)
# NOTE(kgriffs): Set the claim field for
# the active message batch, while also
# filtering out any messages that happened

26
zaqar/storage/mongodb/messages.py
View File

@ -135,6 +135,7 @@ class MessageController(storage.Message):
claim -> c
client uuid -> u
transaction -> tx
delay -> d
"""
def __init__(self, *args, **kwargs):
@ -230,7 +231,8 @@ class MessageController(storage.Message):
def _list(self, queue_name, project=None, marker=None,
echo=False, client_uuid=None, projection=None,
include_claimed=False, sort=1, limit=None):
include_claimed=False, include_delayed=False,
sort=1, limit=None):
"""Message document listing helper.
:param queue_name: Name of the queue to list
@ -248,6 +250,8 @@ class MessageController(storage.Message):
include or exclude
:param include_claimed: (Default False) Whether to include
claimed messages, not just active ones
:param include_delayed: (Default False) Whether to include
delayed messages, not just active ones
:param sort: (Default 1) Sort order for the listing. Pass 1 for
ascending (oldest message first), or -1 for descending (newest
message first).
@ -290,6 +294,14 @@ class MessageController(storage.Message):
# any claim, or are part of an expired claim.
query['c.e'] = {'$lte': now}
if not include_delayed:
# NOTE(cdyangzhenyu): Only include messages that are not
# part of any delay, or are part of an expired delay. if
# the message has no attribute 'd', it will also be obtained.
# This is for compatibility with old data.
query['$or'] = [{'d': {'$lte': now}},
{'d': {'$exists': False}}]
# Construct the request
cursor = collection.find(query,
projection=projection,
@ -338,12 +350,12 @@ class MessageController(storage.Message):
def _active(self, queue_name, marker=None, echo=False,
client_uuid=None, projection=None, project=None,
limit=None):
limit=None, include_delayed=False):
return self._list(queue_name, project=project, marker=marker,
echo=echo, client_uuid=client_uuid,
projection=projection, include_claimed=False,
limit=limit)
include_delayed=include_delayed, limit=limit)
def _claimed(self, queue_name, claim_id,
expires=None, limit=None, project=None):
@ -517,7 +529,8 @@ class MessageController(storage.Message):
def list(self, queue_name, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None, include_claimed=False):
echo=False, client_uuid=None, include_claimed=False,
include_delayed=False):
if marker is not None:
try:
@ -527,7 +540,8 @@ class MessageController(storage.Message):
messages = self._list(queue_name, project=project, marker=marker,
client_uuid=client_uuid, echo=echo,
include_claimed=include_claimed, limit=limit)
include_claimed=include_claimed,
include_delayed=include_delayed, limit=limit)
marker_id = {}
@ -638,6 +652,7 @@ class MessageController(storage.Message):
'e': now_dt + datetime.timedelta(seconds=message['ttl']),
'u': client_uuid,
'c': {'id': None, 'e': now, 'c': 0},
'd': now + message.get('delay', 0),
'b': message['body'] if 'body' in message else {},
'k': next_marker + index,
'tx': None,
@ -815,6 +830,7 @@ class FIFOMessageController(MessageController):
'e': now_dt + datetime.timedelta(seconds=message['ttl']),
'u': client_uuid,
'c': {'id': None, 'e': now, 'c': 0},
'd': now + message.get('delay', 0),
'b': message['body'] if 'body' in message else {},
'k': next_marker + index,
'tx': transaction,

6
zaqar/storage/pooling.py
View File

@ -305,13 +305,15 @@ class MessageController(storage.Message):
def list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None, include_claimed=False):
echo=False, client_uuid=None, include_claimed=False,
include_delayed=False):
control = self._get_controller(queue, project)
if control:
return control.list(queue, project=project,
marker=marker, limit=limit,
echo=echo, client_uuid=client_uuid,
include_claimed=include_claimed)
include_claimed=include_claimed,
include_delayed=include_delayed)
return iter([[]])
def get(self, queue, message_id, project=None):

13
zaqar/tests/unit/transport/wsgi/v2_0/test_queue_lifecycle.py
View File

@ -109,6 +109,7 @@ class TestQueueLifecycleMongoDB(base.V2Base):
ref_doc = jsonutils.loads(doc)
ref_doc['_default_message_ttl'] = 3600
ref_doc['_max_messages_post_size'] = 262144
ref_doc['_default_message_delay'] = 0
self.assertEqual(ref_doc, result_doc)
# Stats empty queue
@ -186,6 +187,8 @@ class TestQueueLifecycleMongoDB(base.V2Base):
result_doc.get('_max_messages_post_size'))
self.assertEqual(3600,
result_doc.get('_default_message_ttl'))
self.assertEqual(0,
result_doc.get('_default_message_delay'))
@ddt.data('{', '[]', '.', ' ')
def test_bad_metadata(self, document):
@ -243,6 +246,7 @@ class TestQueueLifecycleMongoDB(base.V2Base):
ref_doc = jsonutils.loads(doc)
ref_doc['_default_message_ttl'] = 3600
ref_doc['_max_messages_post_size'] = 262144
ref_doc['_default_message_delay'] = 0
self.assertEqual(ref_doc, result_doc)
self.assertEqual(falcon.HTTP_200, self.srmock.status)
@ -296,7 +300,8 @@ class TestQueueLifecycleMongoDB(base.V2Base):
result_doc = jsonutils.loads(result[0])
self.assertEqual({'key1': 2, 'key2': 1,
'_default_message_ttl': 300,
'_max_messages_post_size': 262144}, result_doc)
'_max_messages_post_size': 262144,
'_default_message_delay': 0}, result_doc)
# remove metadata
doc3 = '[{"op":"remove", "path": "/metadata/key1"}]'
@ -317,7 +322,8 @@ class TestQueueLifecycleMongoDB(base.V2Base):
headers=headers)
result_doc = jsonutils.loads(result[0])
self.assertEqual({'key2': 1, '_default_message_ttl': 3600,
'_max_messages_post_size': 262144}, result_doc)
'_max_messages_post_size': 262144,
'_default_message_delay': 0}, result_doc)
# replace non-existent metadata
doc4 = '[{"op":"replace", "path": "/metadata/key3", "value":2}]'
@ -431,7 +437,8 @@ class TestQueueLifecycleMongoDB(base.V2Base):
result_doc = jsonutils.loads(result[0])
self.assertEqual(queue['metadata'], result_doc)
self.assertEqual({'node': 31, '_default_message_ttl': 3600,
'_max_messages_post_size': 262144}, result_doc)
'_max_messages_post_size': 262144,
'_default_message_delay': 0}, result_doc)
# List tail
self.simulate_get(target, headers=header, query_string=params)

3
zaqar/transport/base.py
View File

@ -30,6 +30,9 @@ _GENERAL_TRANSPORT_OPTIONS = (
_RESOURCE_DEFAULTS = (
cfg.IntOpt('default_message_ttl', default=3600,
help=('Defines how long a message will be accessible.')),
cfg.IntOpt('default_message_delay', default=0,
help=('Defines the defautl value for queue delay seconds.'
'The 0 means the delayed queues feature is close.')),
cfg.IntOpt('default_claim_ttl', default=300,
help=('Defines how long a message will be in claimed state.')),
cfg.IntOpt('default_claim_grace', default=60,

30
zaqar/transport/validation.py
View File

@ -28,6 +28,7 @@ from zaqar.i18n import _
MIN_MESSAGE_TTL = 60
MIN_CLAIM_TTL = 60
MIN_CLAIM_GRACE = 60
MIN_DELAY_TTL = 0
MIN_SUBSCRIPTION_TTL = 60
_PURGBLE_RESOURCE_TYPES = {'messages', 'subscriptions'}
@ -68,6 +69,9 @@ _TRANSPORT_LIMITS_OPTIONS = (
deprecated_group='limits:transport',
help='Maximum amount of time a message will be available.'),
cfg.IntOpt('max_message_delay', default=900,
help='Maximum delay seconds for messages can be claimed.'),
cfg.IntOpt('max_claim_ttl', default=43200,
deprecated_name='claim_ttl_max',
deprecated_group='limits:transport',
@ -389,6 +393,21 @@ class Validator(object):
raise ValidationFailed(msg, self._limits_conf.max_message_ttl,
MIN_MESSAGE_TTL)
queue_delay = queue_metadata.get('_default_message_delay',
None)
if queue_delay and not isinstance(queue_delay, int):
msg = _(u'_default_message_delay must be integer.')
raise ValidationFailed(msg)
if queue_delay is not None:
if not (MIN_DELAY_TTL <= queue_delay <=
self._limits_conf.max_message_delay):
msg = _(u'The TTL can not exceed {0} seconds, and must '
'be at least {1} seconds long.')
raise ValidationFailed(
msg, self._limits_conf.max_message_delay,
MIN_DELAY_TTL)
self._validate_retry_policy(queue_metadata)
def queue_purging(self, document):
@ -466,6 +485,17 @@ class Validator(object):
raise ValidationFailed(
msg, self._limits_conf.max_message_ttl, MIN_MESSAGE_TTL)
delay = message.get('delay', 0)
if not (MIN_DELAY_TTL <= delay <=
self._limits_conf.max_message_delay):
msg = _(u'The Delay TTL for a message may not exceed {0} seconds,'
'and must be at least {1} seconds long.')
raise ValidationFailed(
msg, self._limits_conf.max_message_delay,
MIN_DELAY_TTL)
def message_listing(self, limit=None, **kwargs):
"""Restrictions involving a list of messages.

21
zaqar/transport/wsgi/v2_0/messages.py
View File

@ -92,8 +92,26 @@ class CollectionResource(object):
req.get_param_as_int('limit', store=kwargs)
req.get_param_as_bool('echo', store=kwargs)
req.get_param_as_bool('include_claimed', store=kwargs)
req.get_param_as_bool('include_delayed', store=kwargs)
try:
queue_meta = {}
try:
# NOTE(cdyangzhenyu): In order to determine whether the
# queue has a delay attribute, the metadata of the queue
# is obtained here. This may have a little performance impact.
# So maybe a refactor is needed in the future.
queue_meta = self._queue_controller.get_metadata(queue_name,
project_id)
except storage_errors.DoesNotExist as ex:
LOG.exception(ex)
queue_delay = queue_meta.get('_default_message_delay')
if not queue_delay:
# NOTE(cdyangzhenyu): If the queue without the metadata
# attribute _default_message_delay, we don't filter
# for delay messages.
kwargs['include_delayed'] = True
self._validate.message_listing(**kwargs)
results = self._message_controller.list(
queue_name,
@ -168,6 +186,7 @@ class CollectionResource(object):
queue_max_msg_size = queue_meta.get('_max_messages_post_size')
queue_default_ttl = queue_meta.get('_default_message_ttl')
queue_delay = queue_meta.get('_default_message_delay')
if queue_default_ttl:
message_post_spec = (('ttl', int, queue_default_ttl),
@ -175,6 +194,8 @@ class CollectionResource(object):
else:
message_post_spec = (('ttl', int, self._default_message_ttl),
('body', '*', None),)
if queue_delay:
message_post_spec += (('delay', int, queue_delay),)
# Place JSON size restriction before parsing
self._validate.message_length(req.content_length,
max_msg_post_size=queue_max_msg_size)

3
zaqar/transport/wsgi/v2_0/queues.py
View File

@ -30,7 +30,8 @@ LOG = logging.getLogger(__name__)
def _get_reserved_metadata(validate):
_reserved_metadata = ['max_messages_post_size', 'default_message_ttl']
_reserved_metadata = ['max_messages_post_size', 'default_message_ttl',
'default_message_delay']
reserved_metadata = {
'_%s' % meta:
validate.get_limit_conf_value(meta)