openstack/deb-python-kafka
Code Issues Proposed changes

Merge pull request #177 from zever/fetch-last-known-offsets

Browse Source 
Move fetching last known offset logic to a stand alone function.
This commit is contained in:
Dana Powers
2014-08-25 13:22:12 -07:00
parent d73d1690b1 02007b0e80
commit a1603619d4
1 changed files with 17 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
kafka/consumer.py
View File

@@ -100,6 +100,16 @@ class Consumer(object):
self.commit)
self.commit_timer.start()
if auto_commit:
self.fetch_last_known_offsets(partitions)
else:
for partition in partitions:
self.offsets[partition] = 0
def fetch_last_known_offsets(self, partitions=None):
if not partitions:
partitions = self.client.topic_partitions[self.topic]
def get_or_init_offset_callback(resp):
try:
kafka.common.check_error(resp)
@@ -107,16 +117,13 @@ class Consumer(object):
except kafka.common.UnknownTopicOrPartitionError:
return 0
if auto_commit:
for partition in partitions:
req = OffsetFetchRequest(topic, partition)
(offset,) = self.client.send_offset_fetch_request(group, [req],
callback=get_or_init_offset_callback,
fail_on_error=False)
self.offsets[partition] = offset
else:
for partition in partitions:
self.offsets[partition] = 0
for partition in partitions:
req = OffsetFetchRequest(self.topic, partition)
(offset,) = self.client.send_offset_fetch_request(self.group, [req],
callback=get_or_init_offset_callback,
fail_on_error=False)
self.offsets[partition] = offset
self.fetch_offsets = self.offsets.copy()
def commit(self, partitions=None):
"""