openstack/deb-python-kafka
Code Issues Proposed changes

Update kafka-src to latest 0.8

Browse Source 
Fix a broken test (100k was too much to send in one batch)
This commit is contained in:
David Arthur
2013-04-01 22:06:20 -04:00
parent fa13fa88d8
commit 1c5081e6b3
4 changed files with 18 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
kafka-src

Submodule kafka-src updated: 30f992c52d...3c27988ca4

2
kafka/client.py
View File

@@ -116,7 +116,7 @@ class KafkaClient(object):
for payload in payloads: for payload in payloads:
payloads_by_broker[self._get_leader_for_partition(payload.topic, payload.partition)].append(payload) payloads_by_broker[self._get_leader_for_partition(payload.topic, payload.partition)].append(payload)
original_keys.append((payload.topic, payload.partition)) original_keys.append((payload.topic, payload.partition))
# Accumulate the responses in a dictionary # Accumulate the responses in a dictionary
acc = {} acc = {}

3
kafka/conn.py
View File

@@ -71,8 +71,9 @@ class KafkaConnection(object):
def send(self, requestId, payload): def send(self, requestId, payload):
"Send a request to Kafka" "Send a request to Kafka"
log.debug("About to send %d bytes to Kafka" % len(payload))
sent = self._sock.sendall(payload) sent = self._sock.sendall(payload)
if sent == 0: if sent != None:
raise RuntimeError("Kafka went away") raise RuntimeError("Kafka went away")
self.data = self._consume_response() self.data = self._consume_response()

17
test/integration.py
View File

@@ -225,14 +225,25 @@ class TestKafkaClient(unittest.TestCase):
def test_produce_100k_gzipped(self): def test_produce_100k_gzipped(self):
produce = ProduceRequest("test_produce_100k_gzipped", 0, messages=[ req1 = ProduceRequest("test_produce_100k_gzipped", 0, messages=[
create_gzip_message(["Gzipped %d" % i for i in range(100000)]) create_gzip_message(["Gzipped batch 1, message %d" % i for i in range(50000)])
]) ])
for resp in self.client.send_produce_request([produce]): for resp in self.client.send_produce_request([req1]):
self.assertEquals(resp.error, 0) self.assertEquals(resp.error, 0)
self.assertEquals(resp.offset, 0) self.assertEquals(resp.offset, 0)
(offset, ) = self.client.send_offset_request([OffsetRequest("test_produce_100k_gzipped", 0, -1, 1)])
self.assertEquals(offset.offsets[0], 50000)
req2 = ProduceRequest("test_produce_100k_gzipped", 0, messages=[
create_gzip_message(["Gzipped batch 2, message %d" % i for i in range(50000)])
])
for resp in self.client.send_produce_request([req2]):
self.assertEquals(resp.error, 0)
self.assertEquals(resp.offset, 50000)
(offset, ) = self.client.send_offset_request([OffsetRequest("test_produce_100k_gzipped", 0, -1, 1)]) (offset, ) = self.client.send_offset_request([OffsetRequest("test_produce_100k_gzipped", 0, -1, 1)])
self.assertEquals(offset.offsets[0], 100000) self.assertEquals(offset.offsets[0], 100000)