openstack/deb-python-kafka
Code Issues Proposed changes

flake8 pass (pep8 and pyflakes)

Browse Source
This commit is contained in:
mrtheb
2013-10-03 22:52:04 -04:00
parent b0cacc9485
commit a03f0c86b8
6 changed files with 97 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
kafka/client.py
View File

@@ -1,14 +1,9 @@
import base64
from collections import defaultdict from collections import defaultdict
from functools import partial from functools import partial
from itertools import count, cycle
import logging import logging
from operator import attrgetter
import struct
import time import time
import zlib
from kafka.common import * from kafka.common import count, ErrorMapping, TopicAndPartition
from kafka.conn import KafkaConnection from kafka.conn import KafkaConnection
from kafka.protocol import KafkaProtocol from kafka.protocol import KafkaProtocol
@@ -212,8 +207,10 @@ class KafkaClient(object):
order of input payloads order of input payloads
""" """
encoder = partial(KafkaProtocol.encode_produce_request, encoder = partial(
acks=acks, timeout=timeout) KafkaProtocol.encode_produce_request,
acks=acks,
timeout=timeout)
if acks == 0: if acks == 0:
decoder = None decoder = None
@@ -226,9 +223,9 @@ class KafkaClient(object):
for resp in resps: for resp in resps:
# Check for errors # Check for errors
if fail_on_error is True and resp.error != ErrorMapping.NO_ERROR: if fail_on_error is True and resp.error != ErrorMapping.NO_ERROR:
raise Exception("ProduceRequest for %s failed with " raise Exception(
"errorcode=%d" % ( "ProduceRequest for %s failed with errorcode=%d" %
TopicAndPartition(resp.topic, resp.partition), (TopicAndPartition(resp.topic, resp.partition),
resp.error)) resp.error))
# Run the callback # Run the callback
@@ -251,16 +248,17 @@ class KafkaClient(object):
max_wait_time=max_wait_time, max_wait_time=max_wait_time,
min_bytes=min_bytes) min_bytes=min_bytes)
resps = self._send_broker_aware_request(payloads, encoder, resps = self._send_broker_aware_request(
payloads, encoder,
KafkaProtocol.decode_fetch_response) KafkaProtocol.decode_fetch_response)
out = [] out = []
for resp in resps: for resp in resps:
# Check for errors # Check for errors
if fail_on_error is True and resp.error != ErrorMapping.NO_ERROR: if fail_on_error is True and resp.error != ErrorMapping.NO_ERROR:
raise Exception("FetchRequest for %s failed with " raise Exception(
"errorcode=%d" % ( "FetchRequest for %s failed with errorcode=%d" %
TopicAndPartition(resp.topic, resp.partition), (TopicAndPartition(resp.topic, resp.partition),
resp.error)) resp.error))
# Run the callback # Run the callback
@@ -272,7 +270,8 @@ class KafkaClient(object):
def send_offset_request(self, payloads=[], fail_on_error=True, def send_offset_request(self, payloads=[], fail_on_error=True,
callback=None): callback=None):
resps = self._send_broker_aware_request(payloads, resps = self._send_broker_aware_request(
payloads,
KafkaProtocol.encode_offset_request, KafkaProtocol.encode_offset_request,
KafkaProtocol.decode_offset_response) KafkaProtocol.decode_offset_response)

14
kafka/conn.py
View File

@@ -3,6 +3,8 @@ import socket
import struct import struct
from threading import local from threading import local
from kafka.common import BufferUnderflowError
log = logging.getLogger("kafka") log = logging.getLogger("kafka")
@@ -63,7 +65,9 @@ class KafkaConnection(local):
resp = self._sock.recv(self.bufsize) resp = self._sock.recv(self.bufsize)
log.debug("Read %d bytes from Kafka", len(resp)) log.debug("Read %d bytes from Kafka", len(resp))
if resp == "": if resp == "":
raise BufferUnderflowError("Not enough data to read this response") raise BufferUnderflowError(
"Not enough data to read this response")
total += len(resp) total += len(resp)
yield resp yield resp
@@ -75,9 +79,13 @@ class KafkaConnection(local):
def send(self, request_id, payload): def send(self, request_id, payload):
"Send a request to Kafka" "Send a request to Kafka"
log.debug("About to send %d bytes to Kafka, request %d" % (len(payload), request_id))
log.debug(
"About to send %d bytes to Kafka, request %d" %
(len(payload), request_id))
sent = self._sock.sendall(payload) sent = self._sock.sendall(payload)
if sent != None: if sent is not None:
raise RuntimeError("Kafka went away") raise RuntimeError("Kafka went away")
def recv(self, request_id): def recv(self, request_id):

33
kafka/consumer.py
View File

@@ -8,7 +8,7 @@ from Queue import Empty
from kafka.common import ( from kafka.common import (
ErrorMapping, FetchRequest, ErrorMapping, FetchRequest,
OffsetRequest, OffsetFetchRequest, OffsetCommitRequest, OffsetRequest, OffsetCommitRequest,
ConsumerFetchSizeTooSmall, ConsumerNoMoreData ConsumerFetchSizeTooSmall, ConsumerNoMoreData
) )
@@ -223,7 +223,8 @@ class SimpleConsumer(Consumer):
self.fetch_min_bytes = fetch_size_bytes self.fetch_min_bytes = fetch_size_bytes
self.fetch_started = defaultdict(bool) # defaults to false self.fetch_started = defaultdict(bool) # defaults to false
super(SimpleConsumer, self).__init__(client, group, topic, super(SimpleConsumer, self).__init__(
client, group, topic,
partitions=partitions, partitions=partitions,
auto_commit=auto_commit, auto_commit=auto_commit,
auto_commit_every_n=auto_commit_every_n, auto_commit_every_n=auto_commit_every_n,
@@ -275,8 +276,8 @@ class SimpleConsumer(Consumer):
resps = self.client.send_offset_request(reqs) resps = self.client.send_offset_request(reqs)
for resp in resps: for resp in resps:
self.offsets[resp.partition] = resp.offsets[0] + \ self.offsets[resp.partition] = \
deltas[resp.partition] resp.offsets[0] + deltas[resp.partition]
else: else:
raise ValueError("Unexpected value for `whence`, %d" % whence) raise ValueError("Unexpected value for `whence`, %d" % whence)
@@ -364,7 +365,8 @@ class SimpleConsumer(Consumer):
req = FetchRequest( req = FetchRequest(
self.topic, partition, offset, self.client.bufsize) self.topic, partition, offset, self.client.bufsize)
(resp,) = self.client.send_fetch_request([req], (resp,) = self.client.send_fetch_request(
[req],
max_wait_time=self.fetch_max_wait_time, max_wait_time=self.fetch_max_wait_time,
min_bytes=fetch_size) min_bytes=fetch_size)
@@ -376,18 +378,22 @@ class SimpleConsumer(Consumer):
for message in resp.messages: for message in resp.messages:
next_offset = message.offset next_offset = message.offset
# update the offset before the message is yielded. This is # update the offset before the message is yielded. This
# so that the consumer state is not lost in certain cases. # is so that the consumer state is not lost in certain
# For eg: the message is yielded and consumed by the caller, # cases.
# but the caller does not come back into the generator again. #
# The message will be consumed but the status will not be # For eg: the message is yielded and consumed by the
# updated in the consumer # caller, but the caller does not come back into the
# generator again. The message will be consumed but the
# status will not be updated in the consumer
self.fetch_started[partition] = True self.fetch_started[partition] = True
self.offsets[partition] = message.offset self.offsets[partition] = message.offset
yield message yield message
except ConsumerFetchSizeTooSmall, e: except ConsumerFetchSizeTooSmall, e:
log.warn("Fetch size is too small, increasing by 1.5x and retrying")
fetch_size *= 1.5 fetch_size *= 1.5
log.warn(
"Fetch size too small, increasing to %d (1.5x) and retry",
fetch_size)
continue continue
except ConsumerNoMoreData, e: except ConsumerNoMoreData, e:
log.debug("Iteration was ended by %r", e) log.debug("Iteration was ended by %r", e)
@@ -429,7 +435,8 @@ class MultiProcessConsumer(Consumer):
num_procs=1, partitions_per_proc=0): num_procs=1, partitions_per_proc=0):
# Initiate the base consumer class # Initiate the base consumer class
super(MultiProcessConsumer, self).__init__(client, group, topic, super(MultiProcessConsumer, self).__init__(
client, group, topic,
partitions=None, partitions=None,
auto_commit=auto_commit, auto_commit=auto_commit,
auto_commit_every_n=auto_commit_every_n, auto_commit_every_n=auto_commit_every_n,

12
kafka/protocol.py
View File

@@ -120,8 +120,8 @@ class KafkaProtocol(object):
yield OffsetAndMessage(offset, message) yield OffsetAndMessage(offset, message)
except BufferUnderflowError: except BufferUnderflowError:
if read_message is False: if read_message is False:
# If we get a partial read of a message, but haven't yielded anyhting # If we get a partial read of a message, but haven't
# there's a problem # yielded anyhting there's a problem
raise ConsumerFetchSizeTooSmall() raise ConsumerFetchSizeTooSmall()
else: else:
raise StopIteration() raise StopIteration()
@@ -385,15 +385,15 @@ class KafkaProtocol(object):
((partition_error_code, partition, leader, numReplicas), cur) = \ ((partition_error_code, partition, leader, numReplicas), cur) = \
relative_unpack('>hiii', data, cur) relative_unpack('>hiii', data, cur)
(replicas, cur) = relative_unpack('>%di' % numReplicas, (replicas, cur) = relative_unpack(
data, cur) '>%di' % numReplicas, data, cur)
((num_isr,), cur) = relative_unpack('>i', data, cur) ((num_isr,), cur) = relative_unpack('>i', data, cur)
(isr, cur) = relative_unpack('>%di' % num_isr, data, cur) (isr, cur) = relative_unpack('>%di' % num_isr, data, cur)
partition_metadata[partition] = \ partition_metadata[partition] = \
PartitionMetadata(topic_name, partition, leader, PartitionMetadata(
replicas, isr) topic_name, partition, leader, replicas, isr)
topic_metadata[topic_name] = partition_metadata topic_metadata[topic_name] = partition_metadata

17
kafka/queue.py
View File

@@ -25,7 +25,8 @@ class KafkaConsumerProcess(Process):
Process.__init__(self) Process.__init__(self)
def __str__(self): def __str__(self):
return "[KafkaConsumerProcess: topic=%s, partition=%s, sleep=%s]" % \ return "[KafkaConsumerProcess: topic=%s, \
partition=%s, sleep=%s]" % \
(self.topic, self.partition, self.consumer_sleep) (self.topic, self.partition, self.consumer_sleep)
def run(self): def run(self):
@@ -70,10 +71,12 @@ class KafkaProducerProcess(Process):
Process.__init__(self) Process.__init__(self)
def __str__(self): def __str__(self):
return "[KafkaProducerProcess: topic=%s, flush_buffer=%s, \ return "[KafkaProducerProcess: topic=%s, \
flush_timeout=%s, timeout=%s]" % ( flush_buffer=%s, flush_timeout=%s, timeout=%s]" % \
self.topic, self.producer_flush_buffer, (self.topic,
self.producer_flush_timeout, self.producer_timeout) self.producer_flush_buffer,
self.producer_flush_timeout,
self.producer_timeout)
def run(self): def run(self):
self.barrier.wait() self.barrier.wait()
@@ -104,8 +107,8 @@ class KafkaProducerProcess(Process):
last_produce = time.time() last_produce = time.time()
try: try:
msg = KafkaClient.create_message(self.in_queue.get(True, msg = KafkaClient.create_message(
self.producer_timeout)) self.in_queue.get(True, self.producer_timeout))
messages.append(msg) messages.append(msg)
except Empty: except Empty:

6
kafka/util.py
View File

@@ -1,9 +1,8 @@
from collections import defaultdict from collections import defaultdict
from itertools import groupby
import struct import struct
from threading import Thread, Event from threading import Thread, Event
from common import * from kafka.common import BufferUnderflowError
def write_int_string(s): def write_int_string(s):
@@ -39,7 +38,8 @@ def read_short_string(data, cur):
def read_int_string(data, cur): def read_int_string(data, cur):
if len(data) < cur + 4: if len(data) < cur + 4:
raise BufferUnderflowError( raise BufferUnderflowError(
"Not enough data left to read string len (%d < %d)" % (len(data), cur + 4)) "Not enough data left to read string len (%d < %d)" %
(len(data), cur + 4))
(strlen,) = struct.unpack('>i', data[cur:cur + 4]) (strlen,) = struct.unpack('>i', data[cur:cur + 4])
if strlen == -1: if strlen == -1: