openstack/sahara-image-elements
Code Issues Proposed changes

Add different version builders support for Cloudera plugin

Browse Source 
We add 5.0 and 5.3 versions support Cloudera plugin. They will only pull packages
from 5.0.0 and 5.3.0 repositories depending on the version. 5.0.0 will pull less
packages than 5.3.0, for in CDH5.0 plugin we does not support as many services as
in CDH5.3 plugin.

implements bp: cdh-version-management
Change-Id: I720c90aa0f0174148c3ed1eab7880e924ac6835a
This commit is contained in:
Ken Chen 2015-01-18 20:42:16 +08:00
parent 3e13a0cadc
commit 6900b9dcbe
4 changed files with 188 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
diskimage-create/diskimage-create.sh
View File

@ -23,7 +23,7 @@ usage() {
echo "Usage: $(basename $0)" echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]" echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]"
echo " [-i ubuntu|fedora|centos]" echo " [-i ubuntu|fedora|centos]"
echo " [-v 1|2|2.3|2.4|2.6|plain]" echo " [-v 1|2|2.3|2.4|2.6|5.0|5.3|plain]"
echo " [-r 3.1.1|4.0.1]" echo " [-r 3.1.1|4.0.1]"
echo " [-d]" echo " [-d]"
echo " [-m]" echo " [-m]"
@ -113,9 +113,11 @@ fi
if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then
if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.3" -a "$HADOOP_VERSION" != "2.4" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.3" -a "$HADOOP_VERSION" != "2.4" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then
if [ "$PLUGIN" = "cloudera" -a "$HADOOP_VERSION" != "5.0" -a "$HADOOP_VERSION" != "5.3" ]; then
echo -e "Unknown hadoop version selected.\nAborting" echo -e "Unknown hadoop version selected.\nAborting"
exit 1 exit 1
fi fi
fi
fi fi
if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" = "plain" ]; then if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" = "plain" ]; then
@ -493,11 +495,15 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "hdp" ]; then
unset BASE_IMAGE_FILE DIB_IMAGE_SIZE DIB_CLOUD_IMAGES unset BASE_IMAGE_FILE DIB_IMAGE_SIZE DIB_CLOUD_IMAGES
fi fi
#########################
# Images for CDH plugin #
#########################
if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
echo "For cloudera plugin option -v is ignored"
export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"} export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"}
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
cloudera_ubuntu_image_name=${cloudera_ubuntu_image_name:-ubuntu_sahara_cloudera_latest} if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0}
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera" cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
if [ -n "$USE_MIRRORS" ]; then if [ -n "$USE_MIRRORS" ]; then
@ -505,31 +511,70 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
fi fi
# Cloudera supports only 12.04 Ubuntu # Cloudera supports only 12.04 Ubuntu
export DIB_CDH_VERSION="5.0"
export DIB_RELEASE="precise" export DIB_RELEASE="precise"
disk-image-create $cloudera_elements_sequence -o $cloudera_5_0_ubuntu_image_name
mv $cloudera_5_0_ubuntu_image_name.qcow2 ../
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0}
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
disk-image-create $cloudera_elements_sequence -o $cloudera_ubuntu_image_name if [ -n "$USE_MIRRORS" ]; then
mv $cloudera_ubuntu_image_name.qcow2 ../ [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
unset DIB_RELEASE fi
# Cloudera supports only 12.04 Ubuntu
export DIB_CDH_VERSION="5.3"
export DIB_RELEASE="precise"
disk-image-create $cloudera_elements_sequence -o $cloudera_5_3_ubuntu_image_name
mv $cloudera_5_3_ubuntu_image_name.qcow2 ../
unset DIB_CDH_VERSION DIB_RELEASE
fi
fi fi
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
# CentOS cloud image: # CentOS cloud image:
# - Disable including 'base' element for CentOS # - Disable including 'base' element for CentOS
# - Export link and filename for CentOS cloud image to download # - Export link and filename for CentOS cloud image to download
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2" export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
export DIB_CDH_VERSION="5.0"
cloudera_centos_image_name=${cloudera_centos_image_name:-centos_sahara_cloudera_latest} cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0}
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive" cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
if [ -n "$USE_MIRRORS"]; then if [ -n "$USE_MIRRORS"]; then
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror" [ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
fi fi
disk-image-create $cloudera_elements_sequence -n -o $cloudera_centos_image_name disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_0_centos_image_name
mv $cloudera_centos_image_name.qcow2 ../ mv $cloudera_5_0_centos_image_name.qcow2 ../
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
# CentOS cloud image:
# - Disable including 'base' element for CentOS
# - Export link and filename for CentOS cloud image to download
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
export DIB_CDH_VERSION="5.3"
cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0}
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
if [ -n "$USE_MIRRORS"]; then
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
fi
disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_3_centos_image_name
mv $cloudera_5_3_centos_image_name.qcow2 ../
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION
fi
fi fi
unset EXTJS_DOWNLOAD_URL unset EXTJS_DOWNLOAD_URL
fi fi

20
elements/hadoop-cloudera/install.d/50-install-cloudera
View File

@ -12,7 +12,6 @@ install-packages \
cloudera-manager-daemons \ cloudera-manager-daemons \
cloudera-manager-server \ cloudera-manager-server \
cloudera-manager-server-db-2 \ cloudera-manager-server-db-2 \
flume-ng \
hadoop-hdfs-datanode \ hadoop-hdfs-datanode \
hadoop-hdfs-namenode \ hadoop-hdfs-namenode \
hadoop-hdfs-secondarynamenode \ hadoop-hdfs-secondarynamenode \
@ -21,24 +20,31 @@ install-packages \
hadoop-yarn-nodemanager \ hadoop-yarn-nodemanager \
hadoop-yarn-resourcemanager \ hadoop-yarn-resourcemanager \
hbase \ hbase \
hbase-solr \
hive-hcatalog \ hive-hcatalog \
hive-metastore \ hive-metastore \
hive-server2 \ hive-server2 \
hive-webhcat-server \ hive-webhcat-server \
hue \ hue \
ntp \
oozie \
oracle-j2sdk1.7 \
spark-core \
zookeeper
if [ $DIB_CDH_VERSION != "5.0" ]; then
# CDH5.0 does not have below packages.
install-packages \
flume-ng \
hbase-solr \
impala \ impala \
impala-server \ impala-server \
impala-state-store \ impala-state-store \
impala-catalog \ impala-catalog \
ntp \
oozie \
oracle-j2sdk1.7 \
sentry \ sentry \
solr-server \ solr-server \
spark-history-server \ spark-history-server \
sqoop2 \ sqoop2
zookeeper fi
if [ $(lsb_release -is) = 'Ubuntu' ]; then if [ $(lsb_release -is) = 'Ubuntu' ]; then
unset RUNLEVEL unset RUNLEVEL

19
elements/hadoop-cloudera/post-install.d/70-turn-services-off
View File

@ -16,9 +16,7 @@ hive-server2 \
hive-webhcat-server \ hive-webhcat-server \
hue \ hue \
oozie \ oozie \
postgresql \ postgresql
solr-server \
spark-history-server
do do
if [ $(lsb_release -is) = 'Ubuntu' ]; then if [ $(lsb_release -is) = 'Ubuntu' ]; then
update-rc.d -f $i remove update-rc.d -f $i remove
@ -27,6 +25,21 @@ do
fi fi
done done
if [ $DIB_CDH_VERSION != "5.0" ]; then
for i in impala-catalog \
impala-server \
impala-state-store \
solr-server \
spark-history-server
do
if [ $(lsb_release -is) = 'Ubuntu' ]; then
update-rc.d -f $i remove
else
chkconfig $i off
fi
done
fi
if [ $(lsb_release -is) = 'CentOS' ]; then if [ $(lsb_release -is) = 'CentOS' ]; then
chkconfig iptables off chkconfig iptables off
chkconfig ip6tables off chkconfig ip6tables off

83
elements/hadoop-cloudera/pre-install.d/10-add-mirror
View File

@ -2,23 +2,92 @@
distro=$(lsb_release -is) distro=$(lsb_release -is)
case $distro in
function add_repo_5_0_0 {
case $distro in
Ubuntu) Ubuntu)
# Add repository with postgresql package (it's dependency of cloudera packages) # Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo # Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories # Cloudera repositories
wget -O /etc/apt/sources.list.d/cdh5.list http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/cloudera.list echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -O /etc/apt/sources.list.d/cm5.list http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/cloudera.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
apt-get update apt-get update
;; ;;
CentOS) CentOS)
wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cm5/redhat/6/x86_64/cm/cloudera-manager.repo echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
yum clean all
;; ;;
esac esac
}
function add_repo_5_3_0 {
case $distro in
Ubuntu)
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
apt-get update
;;
CentOS)
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
yum clean all
;;
esac
}
# Call version-specific script to install the desired version of CDH
if [[ $DIB_CDH_VERSION == "5.0" ]]; then
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_0_0
else
if [[ $DIB_CDH_VERSION == "5.3" ]]; then
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_3_0
else
echo "Invalid CDH Version specified, exiting install."
exit 1
fi
fi