f139eb258d
Currently Kafka driver for an oslo.messaging uses kafka-python==0.9.5 and mostly broken. This package version supports only low level Kafka producer and consumer API which are marked as deprecated now [1]. Using of these interfaces bring a big concern to the message processing, because current KafkaConsumer has not any consuming coordination. This fact causes a message duplication for the several consumers of one topic. This behavior is specific to Ceilometer and services which read and process notifications from other services. New version of kafka-python allows to use async thread safe message producers and coordinated consumers [1]. [1] http://kafka-python.readthedocs.io/en/master/changelog.html#feb-15-2016 The driver is currently experimental, python-kafka<1.0.0 API have major issue described above that can't make the oslo.messaging driver works, so we prefer having a working driver with a non-synced dependencies, that the reverse. Co-Authored-By: Mehdi Abaakouk <sileht@redhat.com> Change-Id: I29862ed7bf56b9d8878fa8e9fb1cbd9d643908a4
53 lines
2.1 KiB
Python
53 lines
2.1 KiB
Python
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from oslo_config import cfg
|
|
|
|
KAFKA_OPTS = [
|
|
cfg.StrOpt('kafka_default_host', default='localhost',
|
|
deprecated_for_removal=True,
|
|
deprecated_reason="Replaced by [DEFAULT]/transport_url",
|
|
help='Default Kafka broker Host'),
|
|
|
|
cfg.PortOpt('kafka_default_port', default=9092,
|
|
deprecated_for_removal=True,
|
|
deprecated_reason="Replaced by [DEFAULT]/transport_url",
|
|
help='Default Kafka broker Port'),
|
|
|
|
cfg.IntOpt('kafka_max_fetch_bytes', default=1024 * 1024,
|
|
help='Max fetch bytes of Kafka consumer'),
|
|
|
|
cfg.IntOpt('kafka_consumer_timeout', default=1.0,
|
|
help='Default timeout(s) for Kafka consumers'),
|
|
|
|
cfg.IntOpt('pool_size', default=10,
|
|
help='Pool Size for Kafka Consumers'),
|
|
|
|
cfg.IntOpt('conn_pool_min_size', default=2,
|
|
help='The pool size limit for connections expiration policy'),
|
|
|
|
cfg.IntOpt('conn_pool_ttl', default=1200,
|
|
help='The time-to-live in sec of idle connections in the pool'),
|
|
|
|
cfg.StrOpt('consumer_group', default="oslo_messaging_consumer",
|
|
help='Group id for Kafka consumer. Consumers in one group '
|
|
'will coordinate message consumption'),
|
|
|
|
cfg.FloatOpt('producer_batch_timeout', default=0.,
|
|
help="Upper bound on the delay for KafkaProducer batching "
|
|
"in seconds"),
|
|
|
|
cfg.IntOpt('producer_batch_size', default=16384,
|
|
help='Size of batch for the producer async send')
|
|
]
|