openstack/deb-python-kafka
Code Issues Proposed changes

Bulk fetch offset partitions in base consumer -- suggested by ecanzonieri

Browse Source
This commit is contained in:
Dana Powers
2015-03-30 16:55:41 -07:00
parent b6d032cc3f
commit 1d252bfc20
1 changed files with 9 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
kafka/consumer/base.py
View File

@@ -83,12 +83,13 @@ class Consumer(object):
if not partitions: if not partitions:
partitions = self.client.get_partition_ids_for_topic(self.topic) partitions = self.client.get_partition_ids_for_topic(self.topic)
for partition in partitions: responses = self.client.send_offset_fetch_request(
(resp,) = self.client.send_offset_fetch_request( self.group,
self.group, [OffsetFetchRequest(self.topic, p) for p in partitions],
[OffsetFetchRequest(self.topic, partition)], fail_on_error=False
fail_on_error=False )
)
for resp in responses:
try: try:
check_error(resp) check_error(resp)
# API spec says server wont set an error here # API spec says server wont set an error here
@@ -98,12 +99,12 @@ class Consumer(object):
# -1 offset signals no commit is currently stored # -1 offset signals no commit is currently stored
if resp.offset == -1: if resp.offset == -1:
self.offsets[partition] = 0 self.offsets[resp.partition] = 0
# Otherwise we committed the stored offset # Otherwise we committed the stored offset
# and need to fetch the next one # and need to fetch the next one
else: else:
self.offsets[partition] = resp.offset self.offsets[resp.partition] = resp.offset
def commit(self, partitions=None): def commit(self, partitions=None):
""" """