openstack/deb-python-kafka
Code Issues Proposed changes

Log warning if message set appears double-compressed in KafkaConsumer

Browse Source
This commit is contained in:
Dana Powers
2016-07-14 23:38:14 -07:00
parent a8c86d379a
commit ed6098c272
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
kafka/consumer/fetcher.py
View File

@@ -361,6 +361,14 @@ class Fetcher(six.Iterator):
# If relative offset is used, we need to decompress the entire message first to compute # If relative offset is used, we need to decompress the entire message first to compute
# the absolute offset. # the absolute offset.
inner_mset = msg.decompress() inner_mset = msg.decompress()
# There should only ever be a single layer of compression
if inner_mset[0][-1].is_compressed():
log.warning('MessageSet at %s offset %d appears '
' double-compressed. This should not'
' happen -- check your producers!',
tp, offset)
if msg.magic > 0: if msg.magic > 0:
last_offset, _, _ = inner_mset[-1] last_offset, _, _ = inner_mset[-1]
absolute_base_offset = offset - last_offset absolute_base_offset = offset - last_offset