openstack/deb-python-kafka
Code Issues Proposed changes

Fix concurrency bug in RecordAccumulator.ready()

Browse Source
This commit is contained in:
Dana Powers
2016-02-18 21:54:12 -08:00
parent 2dd216b5ac
commit f2d10f02d3
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
kafka/producer/record_accumulator.py
View File

@@ -320,8 +320,11 @@ class RecordAccumulator(object):
now = time.time() now = time.time()
exhausted = bool(self._free.queued() > 0) exhausted = bool(self._free.queued() > 0)
for tp, dq in six.iteritems(self._batches): # several threads are accessing self._batches -- to simplify
# concurrent access, we iterate over a snapshot of partitions
# and lock each partition separately as needed
partitions = list(self._batches.keys())
for tp in partitions:
leader = cluster.leader_for_partition(tp) leader = cluster.leader_for_partition(tp)
if leader is None or leader == -1: if leader is None or leader == -1:
unknown_leaders_exist = True unknown_leaders_exist = True
@@ -330,6 +333,7 @@ class RecordAccumulator(object):
continue continue
with self._tp_locks[tp]: with self._tp_locks[tp]:
dq = self._batches[tp]
if not dq: if not dq:
continue continue
batch = dq[0] batch = dq[0]