openstack/sahara-image-elements
Code Issues Proposed changes

Add Kafka 2.0.2 to CDH 5.5 and 5.7

Browse Source 
bp: cdh-kafka-service
Change-Id: Ieef1dc803431872a345e41862726b1676d8253d9
This commit is contained in:
Michael Ionkin 2016-08-16 13:12:18 +03:00
parent 6010b9a4e1
commit cb60bb9ac7
3 changed files with 31 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
elements/hadoop-cloudera/README.rst
View File

@ -6,10 +6,13 @@ Installs cloudera (cloudera-manager-agent cloudera-manager-daemons
cloudera-manager-server cloudera-manager-server-db-2 hadoop-hdfs-namenode
hadoop-hdfs-secondarynamenode hadoop-hdfs-datanode hadoop-yarn-resourcemanager
hadoop-yarn-nodemanager hadoop-mapreduce hadoop-mapreduce-historyserver) and
Java (oracle-j2sdk1.7) packages from cloudera repositories
Java (oracle-j2sdk1.7) packages from cloudera repositories:
`cdh5 <http://archive-primary.cloudera.com/cdh5/>`_ and
`cm5 <http://archive-primary.cloudera.com/cm5>`_.
Also installs Cloudera distribution of Apache Kafka for CDH version >= 5.5 from
Cloudera repository: `kafka <http://archive-primary.cloudera.com/kafka>`_.
In order to create the Cloudera images with ``diskimage-create.sh``, use the
following syntax to select the ``cloudera`` plugin:

7
elements/hadoop-cloudera/install.d/50-install-cloudera
View File

@ -52,6 +52,13 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
spark-history-server \
sqoop2
fi
if [ $DIB_CDH_VERSION \> "5.4" ]; then
# we support Kafka from CDH 5.5
install-packages \
kafka \
kafka-server
fi
fi
case "$DIB_CDH_VERSION" in

20
elements/hadoop-cloudera/pre-install.d/10-add-mirror
View File

@ -150,6 +150,10 @@ function add_repo_5_5_0 {
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add -
# add Kafka repository
echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list
wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add -
#change repository priority
echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin
@ -182,6 +186,12 @@ function add_repo_5_5_0 {
echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo
echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo
echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo
yum clean all
;;
esac
@ -208,6 +218,10 @@ function add_repo_5_7_x {
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add -
# add Kafka repository
echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/ trusty-kafka2.0.2 contrib' >> /etc/apt/sources.list
wget -qO - https://archive.cloudera.com/kafka/ubuntu/trusty/amd64/kafka/archive.key | apt-key add -
#change repository priority
echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin
@ -240,6 +254,12 @@ function add_repo_5_7_x {
echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo
echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo
echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo
yum clean all
;;
esac