Install kafka

Rudimentary install of kafka.

Change-Id: I34da559d4e442ddd909f8329dbdd864fa8e63a32

devstack/files/kafka/kafka-server-start.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+	echo "USAGE: $0 [-daemon] server.properties"
+	exit 1
+fi
+base_dir=$(dirname $0)
+export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$base_dir/../config/log4j.properties"
+export KAFKA_HEAP_OPTS="-Xms256m -Xmx256m"
+
+EXTRA_ARGS="-name kafkaServer -loggc"
+
+COMMAND=$1
+case $COMMAND in
+  -daemon)
+    EXTRA_ARGS="-daemon "$EXTRA_ARGS
+    shift
+    ;;
+  *)
+    ;;
+esac
+
+exec $base_dir/kafka-run-class.sh $EXTRA_ARGS kafka.Kafka $@

devstack/files/kafka/kafka.conf

@@ -0,0 +1,18 @@
+description "Kafka"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+limit nofile 32768 32768
+
+# If zookeeper is running on this box also give it time to start up properly
+pre-start script
+    if [ -e /etc/init.d/zookeeper ]; then
+        /etc/init.d/zookeeper start
+    fi
+end script
+
+# Rather than using setuid/setgid sudo is used because the pre-start task must run as root
+exec sudo -Hu kafka -g kafka KAFKA_HEAP_OPTS="-Xmx256m" /opt/kafka/bin/kafka-server-start.sh /etc/kafka/server.properties

devstack/files/kafka/log4j.properties

@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=WARN, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.kafkaAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.kafkaAppender.MaxFileSize=50MB
+log4j.appender.kafkaAppender.MaxBackupIndex=4
+log4j.appender.kafkaAppender.File=/var/log/kafka/server.log
+log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.stateChangeAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.stateChangeAppender.MaxFileSize=50MB
+log4j.appender.stateChangeAppender.MaxBackupIndex=4
+log4j.appender.stateChangeAppender.File=/var/log/kafka/state-change.log
+log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+log4j.appender.controllerAppender=org.apache.log4j.RollingFileAppender
+log4j.appender.controllerAppender.MaxFileSize=50MB
+log4j.appender.controllerAppender.MaxBackupIndex=4
+log4j.appender.controllerAppender.File=/var/log/kafka/controller.log
+log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
+log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+# Turn on all our debugging info
+#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender
+#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender
+#log4j.logger.kafka.perf=DEBUG, kafkaAppender
+#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender
+#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG
+log4j.logger.kafka=WARN, kafkaAppender
+
+# Tracing requests results in large logs
+#log4j.appender.requestAppender=org.apache.log4j.RollingFileAppender
+#log4j.appender.requestAppender.MaxFileSize=50MB
+#log4j.appender.requestAppender.MaxBackupIndex=4
+#log4j.appender.requestAppender.File=/var/log/kafka/kafka-request.log
+#log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
+#log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
+#
+#log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender
+#log4j.additivity.kafka.network.RequestChannel$=false
+#
+#log4j.logger.kafka.network.Processor=TRACE, requestAppender
+#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
+#log4j.additivity.kafka.server.KafkaApis=false
+#log4j.logger.kafka.request.logger=TRACE, requestAppender
+#log4j.additivity.kafka.request.logger=false
+
+log4j.logger.kafka.controller=TRACE, controllerAppender
+log4j.additivity.kafka.controller=false
+
+log4j.logger.state.change.logger=TRACE, stateChangeAppender
+log4j.additivity.state.change.logger=false
+

devstack/files/kafka/server.properties

@@ -0,0 +1,102 @@
+############################# Server Basics #############################
+
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id=0
+
+############################# Socket Server Settings #############################
+
+# The port the socket server listens on
+port=9092
+
+# Hostname the broker will bind to. If not set, the server will bind to all interfaces
+host.name=127.0.0.1
+
+# Hostname the broker will advertise to producers and consumers. If not set, it uses the
+# value for "host.name" if configured.  Otherwise, it will use the value returned from
+# java.net.InetAddress.getCanonicalHostName().
+#advertised.host.name=<hostname routable by clients>
+
+# The port to publish to ZooKeeper for clients to use. If this is not set,
+# it will publish the same port that the broker binds to.
+#advertised.port=<port accessible by clients>
+
+# The number of threads handling network requests
+num.network.threads=2
+
+# The number of threads doing disk I/O
+num.io.threads=2
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes=1048576
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes=1048576
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes=104857600
+
+
+############################# Log Basics #############################
+
+# A comma seperated list of directories under which to store log files
+log.dirs=/var/kafka
+
+auto.create.topics.enable=false
+# The number of logical partitions per topic per server. More partitions allow greater parallelism
+# for consumption, but also mean more files.
+num.partitions=2
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+#    1. Durability: Unflushed data may be lost if you are not using replication.
+#    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+#    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+log.flush.interval.ms=1000
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion
+log.retention.hours=24
+
+# A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
+# segments don't drop below log.retention.bytes.
+log.retention.bytes=104857600
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+log.segment.bytes=104857600
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms=60000
+
+# By default the log cleaner is disabled and the log retention policy will default to just delete segments after their retention expires.
+# If log.cleaner.enable=true is set the cleaner will be enabled and individual logs can then be marked for log compaction.
+log.cleaner.enable=false
+
+############################# Zookeeper #############################
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect=127.0.0.1:2181
+
+# Timeout in ms for connecting to zookeeper
+zookeeper.connection.timeout.ms=1000000
+

devstack/files/zookeeper/zoo.cfg

@@ -23,7 +23,7 @@ maxClientCnxns=60
 # The fist port is used by followers to connect to the leader
 # The second one is used for leader election
 
-server.0=localhost:2888:3888
+server.0=127.0.0.1:2888:3888
 
 # To avoid seeks ZooKeeper allocates space in the transaction log file in
 # blocks of preAllocSize kilobytes. The default block size is 64M. One reason

devstack/plugin.sh

@@ -49,6 +49,8 @@ function install_monasca {
 
     install_zookeeper
 
+    install_kafka
+
 }
 
 function post_config_monasca {
@@ -65,6 +67,8 @@ function unstack_monasca {
 
 function clean_monasca {
 
+    clean_kafka
+
     clean_zookeeper
 
 }
@@ -117,6 +121,86 @@ function clean_openjdk-7 {
 
 }
 
+function install_kafka {
+
+    install_openjdk-7
+
+    sudo curl http://apache.mirrors.tds.net/kafka/0.8.1.1/kafka_2.9.2-0.8.1.1.tgz -o /root/kafka_2.9.2-0.8.1.1.tgz
+
+    sudo groupadd --system kafka
+
+    sudo useradd --system -g kafka kafka
+
+    sudo tar -xzf /root/kafka_2.9.2-0.8.1.1.tgz -C /opt
+
+    sudo ln -s /opt/kafka_2.9.2-0.8.1.1 /opt/kafka
+
+    sudo cp -f /opt/stack/monasca/devstack/files/kafka/kafka-server-start.sh /opt/kafka_2.9.2-0.8.1.1/bin/kafka-server-start.sh
+
+    sudo cp -f /opt/stack/monasca/devstack/files/kafka/kafka.conf /etc/init/kafka.conf
+
+    sudo chown root:root /etc/init/kafka.conf
+
+    sudo chmod 644 /etc/init/kafka.conf
+
+    sudo mkdir -p /var/kafka
+
+    sudo chown kafka:kafka /var/kafka
+
+    sudo chmod 755 /var/kafka
+
+    sudo rm -rf /var/kafka/lost+found
+
+    sudo mkdir -p /var/log/kafka
+
+    sudo chown kafka:kafka /var/log/kafka
+
+    sudo chmod 755 /var/log/kafka
+
+    sudo ln -s /opt/kafka/config /etc/kafka
+
+    sudo cp -f /opt/stack/monasca/devstack/files/kafka/log4j.properties /etc/kafka/log4j.properties
+
+    sudo chown kafka:kafka /etc/kafka/log4j.properties
+
+    sudo chmod 644 /etc/kafka/log4j.properties
+
+    sudo cp -f /opt/stack/monasca/devstack/files/kafka/server.properties /etc/kafka/server.properties
+
+    sudo chown kafka:kafka /etc/kafka/server.properties
+
+    sudo chmod 644 /etc/kafka/server.properties
+
+    sudo start kafka
+
+}
+
+function clean_kafka {
+
+    sudo stop kafka
+
+    sudo rm -rf /var/kafka
+
+    sudo rm -rf /var/log/kafka
+
+    sudo rm -rf /etc/kafka
+
+    sudo rm -f /opt/kafka
+
+    sudo rm -rf /etc/init/kafka.conf
+
+    sudo userdel kafka
+
+    sudo groupdel kafka
+
+    sudo rm -rf /opt/kafka_2.9.2-0.8.1.1
+
+    sudo rm -rf /root/kafka_2.9.2-0.8.1.1.tgz
+
+    clean_openjdk-7
+
+}
+
 # Allows this script to be called directly outside of
 # the devstack infrastructure code.
 if [[ $(type -t) != 'function' ]]; then