5.3 KiB
5.3 KiB
Simple APIs (DEPRECATED)
SimpleConsumer (DEPRECATED)
from kafka import SimpleProducer, SimpleClient
# To consume messages
= SimpleClient('localhost:9092')
client = SimpleConsumer(client, "my-group", "my-topic")
consumer for message in consumer:
# message is raw byte string -- decode if necessary!
# e.g., for unicode: `message.decode('utf-8')`
print(message)
# Use multiprocessing for parallel consumers
from kafka import MultiProcessConsumer
# This will split the number of partitions among two processes
= MultiProcessConsumer(client, "my-group", "my-topic", num_procs=2)
consumer
# This will spawn processes such that each handles 2 partitions max
= MultiProcessConsumer(client, "my-group", "my-topic",
consumer =2)
partitions_per_proc
for message in consumer:
print(message)
for message in consumer.get_messages(count=5, block=True, timeout=4):
print(message)
client.close()
SimpleProducer (DEPRECATED)
Asynchronous Mode
from kafka import SimpleProducer, SimpleClient
# To send messages asynchronously
= SimpleClient('localhost:9092')
client = SimpleProducer(client, async=True)
producer 'my-topic', b'async message')
producer.send_messages(
# To send messages in batch. You can use any of the available
# producers for doing this. The following producer will collect
# messages in batch and send them to Kafka after 20 messages are
# collected or every 60 seconds
# Notes:
# * If the producer dies before the messages are sent, there will be losses
# * Call producer.stop() to send the messages and cleanup
= SimpleProducer(client,
producer async=True,
=20,
batch_send_every_n=60) batch_send_every_t
Synchronous Mode
from kafka import SimpleProducer, SimpleClient
# To send messages synchronously
= SimpleClient('localhost:9092')
client = SimpleProducer(client, async=False)
producer
# Note that the application is responsible for encoding messages to type bytes
'my-topic', b'some message')
producer.send_messages('my-topic', b'this method', b'is variadic')
producer.send_messages(
# Send unicode message
'my-topic', u'你怎么样?'.encode('utf-8'))
producer.send_messages(
# To wait for acknowledgements
# ACK_AFTER_LOCAL_WRITE : server will wait till the data is written to
# a local log before sending response
# ACK_AFTER_CLUSTER_COMMIT : server will block until the message is committed
# by all in sync replicas before sending a response
= SimpleProducer(client,
producer async=False,
=SimpleProducer.ACK_AFTER_LOCAL_WRITE,
req_acks=2000,
ack_timeout=False)
sync_fail_on_error
= producer.send_messages('my-topic', b'another message')
responses for r in responses:
logging.info(r.offset)
KeyedProducer (DEPRECATED)
from kafka import (
SimpleClient, KeyedProducer,
Murmur2Partitioner, RoundRobinPartitioner)
= SimpleClient('localhost:9092')
kafka
# HashedPartitioner is default (currently uses python hash())
= KeyedProducer(kafka)
producer b'my-topic', b'key1', b'some message')
producer.send_messages(b'my-topic', b'key2', b'this methode')
producer.send_messages(
# Murmur2Partitioner attempts to mirror the java client hashing
= KeyedProducer(kafka, partitioner=Murmur2Partitioner)
producer
# Or just produce round-robin (or just use SimpleProducer)
= KeyedProducer(kafka, partitioner=RoundRobinPartitioner) producer
SimpleClient (DEPRECATED)
import time
from kafka import SimpleClient
from kafka.errors import LeaderNotAvailableError, NotLeaderForPartitionError
from kafka.protocol import create_message
from kafka.structs import ProduceRequestPayload
= SimpleClient('localhost:9092')
kafka = ProduceRequestPayload(topic='my-topic', partition=0,
payload =[create_message("some message")])
messages
= 5
retries = []
resps while retries and not resps:
-= 1
retries try:
= kafka.send_produce_request(
resps =[payload], fail_on_error=True)
payloadsexcept LeaderNotAvailableError, NotLeaderForPartitionError:
kafka.load_metadata_for_topics()1)
time.sleep(
# Other exceptions you might consider handling:
# UnknownTopicOrPartitionError, TopicAuthorizationFailedError,
# RequestTimedOutError, MessageSizeTooLargeError, InvalidTopicError,
# RecordListTooLargeError, InvalidRequiredAcksError,
# NotEnoughReplicasError, NotEnoughReplicasAfterAppendError
kafka.close()
0].topic # 'my-topic'
resps[0].partition # 0
resps[0].error # 0
resps[0].offset # offset of the first message sent in this request resps[