Kafka message max size
Added new variable max_message_size to config file. Do not use the same variable to validate envelope and payload size. Change-Id: I1552ef88a5b54ad8d7b530627f2c54cededd370e
This commit is contained in:
parent
52e66aa17a
commit
310e3053df
|
@ -25,6 +25,9 @@ kafka_url = '192.168.10.6:9092'
|
||||||
# Uncomment for monasca-vagrant environment
|
# Uncomment for monasca-vagrant environment
|
||||||
#kafka_url = '192.168.10.4:9092'
|
#kafka_url = '192.168.10.4:9092'
|
||||||
|
|
||||||
|
#Message max size that can be sent to Kafka
|
||||||
|
max_message_size = 1048576
|
||||||
|
|
||||||
[keystone_authtoken]
|
[keystone_authtoken]
|
||||||
|
|
||||||
# Uncomment for DevStack environment
|
# Uncomment for DevStack environment
|
||||||
|
|
|
@ -26,14 +26,22 @@ from monasca_log_api.reference.common import validation
|
||||||
LOG = log.getLogger(__name__)
|
LOG = log.getLogger(__name__)
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
_MAX_MESSAGE_SIZE = 1048576
|
||||||
|
|
||||||
log_publisher_opts = [
|
log_publisher_opts = [
|
||||||
cfg.StrOpt('kafka_url',
|
cfg.StrOpt('kafka_url',
|
||||||
required=True,
|
required=True,
|
||||||
help='Url to kafka server'),
|
help='Url to kafka server'),
|
||||||
cfg.MultiStrOpt('topics',
|
cfg.MultiStrOpt('topics',
|
||||||
default=['logs'],
|
default=['logs'],
|
||||||
help='Consumer topics')
|
help='Consumer topics'),
|
||||||
|
cfg.IntOpt('max_message_size',
|
||||||
|
default=_MAX_MESSAGE_SIZE,
|
||||||
|
required=True,
|
||||||
|
help=('Message max size that can be sent '
|
||||||
|
'to kafka, default to %d bytes' % _MAX_MESSAGE_SIZE))
|
||||||
]
|
]
|
||||||
|
|
||||||
log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher')
|
log_publisher_group = cfg.OptGroup(name='log_publisher', title='log_publisher')
|
||||||
|
|
||||||
cfg.CONF.register_group(log_publisher_group)
|
cfg.CONF.register_group(log_publisher_group)
|
||||||
|
|
|
@ -227,15 +227,16 @@ def validate_envelope_size(envelope=None):
|
||||||
:param str envelope: serialized envelope
|
:param str envelope: serialized envelope
|
||||||
:exception: :py:class:`falcon.HTTPInternalServerError`
|
:exception: :py:class:`falcon.HTTPInternalServerError`
|
||||||
"""
|
"""
|
||||||
max_size = CONF.service.max_log_size
|
max_size = CONF.log_publisher.max_message_size
|
||||||
envelope_size = sys.getsizeof(envelope) if envelope is not None else -1
|
envelope_size = sys.getsizeof(envelope) if envelope is not None else -1
|
||||||
|
|
||||||
LOG.debug('Envelope size is %s', envelope_size)
|
LOG.debug('Envelope size is %s', envelope_size)
|
||||||
|
|
||||||
if envelope_size >= max_size:
|
if envelope_size >= max_size:
|
||||||
|
error_msg = (envelope_size, max_size)
|
||||||
raise falcon.HTTPInternalServerError(
|
raise falcon.HTTPInternalServerError(
|
||||||
title='Envelope size exceeded',
|
title='Kafka message size exceeded',
|
||||||
description='Maximum allowed size is %d bytes' % max_size
|
description='%d exceeds maximum allowed size %d bytes' % error_msg
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -334,17 +334,17 @@ class EnvelopeSizeValidations(testing.TestBase):
|
||||||
|
|
||||||
def test_should_pass_envelope_size_ok(self):
|
def test_should_pass_envelope_size_ok(self):
|
||||||
envelope = self._rand_str(120)
|
envelope = self._rand_str(120)
|
||||||
max_log_size = 240
|
max_message_size = 240
|
||||||
self.conf.config(max_log_size=max_log_size,
|
self.conf.config(max_message_size=max_message_size,
|
||||||
group='service')
|
group='log_publisher')
|
||||||
|
|
||||||
validation.validate_envelope_size(envelope)
|
validation.validate_envelope_size(envelope)
|
||||||
|
|
||||||
def test_should_pass_envelope_size_exceeded(self):
|
def test_should_pass_envelope_size_exceeded(self):
|
||||||
envelope = self._rand_str(360)
|
envelope = self._rand_str(360)
|
||||||
max_log_size = 240
|
max_message_size = 240
|
||||||
self.conf.config(max_log_size=max_log_size,
|
self.conf.config(max_message_size=max_message_size,
|
||||||
group='service')
|
group='log_publisher')
|
||||||
|
|
||||||
self.assertRaises(
|
self.assertRaises(
|
||||||
errors.HTTPInternalServerError,
|
errors.HTTPInternalServerError,
|
||||||
|
@ -354,9 +354,9 @@ class EnvelopeSizeValidations(testing.TestBase):
|
||||||
|
|
||||||
def test_should_pass_envelope_size_equal(self):
|
def test_should_pass_envelope_size_equal(self):
|
||||||
envelope = self._rand_str(240)
|
envelope = self._rand_str(240)
|
||||||
max_log_size = 240
|
max_message_size = 240
|
||||||
self.conf.config(max_log_size=max_log_size,
|
self.conf.config(max_message_size=max_message_size,
|
||||||
group='service')
|
group='log_publisher')
|
||||||
|
|
||||||
self.assertRaises(
|
self.assertRaises(
|
||||||
errors.HTTPInternalServerError,
|
errors.HTTPInternalServerError,
|
||||||
|
|
Loading…
Reference in New Issue