Merge "[CDH] Support CDH 5.7"
This commit is contained in:
commit
846c42f86f
|
@ -32,7 +32,7 @@ usage() {
|
|||
echo "Usage: $(basename $0)"
|
||||
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
|
||||
echo " [-i ubuntu|fedora|centos|centos7]"
|
||||
echo " [-v 2.6|2.7.1|4|5.0|5.3|5.4|5.5|2.2.0.0|2.2.1.0]"
|
||||
echo " [-v 2.6|2.7.1|4|5.0|5.3|5.4|5.5|5.7|2.2.0.0|2.2.1.0]"
|
||||
echo " [-r 5.0.0|5.1.0]"
|
||||
echo " [-s 1.3.1|1.6.0]"
|
||||
echo " [-t 0.9.2|1.0.1]"
|
||||
|
@ -173,18 +173,23 @@ case "$PLUGIN" in
|
|||
esac
|
||||
|
||||
case "$HADOOP_VERSION" in
|
||||
"" | "5.0" | "5.3" | "5.4" | "5.5");;
|
||||
"" | "5.0" | "5.3" | "5.4" | "5.5" | "5.7");;
|
||||
*)
|
||||
echo -e "Unknown hadoop version selected.\nAborting"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$BASE_IMAGE_OS" = "centos7" ]; then
|
||||
if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" ]; then
|
||||
echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5"
|
||||
if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" -a ! "$HADOOP_VERSION" = "5.7" ]; then
|
||||
echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5 and CDH 5.7"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$DIB_CDH_MINOR_VERSION" ]; then
|
||||
echo -e "Continuing image building with custom CDH version: \"$DIB_CDH_MINOR_VERSION\".\n"
|
||||
fi
|
||||
;;
|
||||
"spark")
|
||||
case "$BASE_IMAGE_OS" in
|
||||
|
@ -657,6 +662,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
|
|||
export plugin_type="cloudera"
|
||||
export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce"
|
||||
|
||||
if [ -n "$DIB_CDH_MINOR_VERSION" ]; then
|
||||
# cut minor version number, e.g. from 5.7.1 to 5.7
|
||||
# this is needed if user specified minor version but didn't specify
|
||||
# hadoop version by '-v' parameter
|
||||
HADOOP_VERSION=${DIB_CDH_MINOR_VERSION%.*}
|
||||
fi
|
||||
export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$HADOOP_VERSION.0}
|
||||
|
||||
cloudera_elements_sequence="hadoop-cloudera swift_hadoop"
|
||||
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
|
||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
|
||||
|
@ -695,6 +708,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
|
|||
image_create ubuntu $cloudera_5_5_ubuntu_image_name $cloudera_elements_sequence
|
||||
unset DIB_CDH_VERSION DIB_RELEASE
|
||||
fi
|
||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then
|
||||
cloudera_5_7_ubuntu_image_name=${cloudera_5_7_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION}
|
||||
|
||||
export DIB_CDH_VERSION="5.7"
|
||||
export DIB_RELEASE="trusty"
|
||||
image_create ubuntu $cloudera_5_7_ubuntu_image_name $cloudera_elements_sequence
|
||||
unset DIB_CDH_VERSION DIB_RELEASE
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then
|
||||
|
@ -741,10 +762,19 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
|
|||
cloudera_5_5_centos7_image_name=${cloudera_5_5_centos7_image_name:-centos7_sahara_cloudera_5_5_0}
|
||||
image_create centos7 $cloudera_5_5_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence
|
||||
|
||||
unset DIB_CDH_VERSION
|
||||
fi
|
||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then
|
||||
export DIB_CDH_VERSION="5.7"
|
||||
|
||||
cloudera_5_7_centos7_image_name=${cloudera_5_7_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION}
|
||||
image_create centos7 $cloudera_5_7_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence
|
||||
|
||||
unset DIB_CDH_VERSION
|
||||
fi
|
||||
fi
|
||||
|
||||
unset DIB_CDH_MINOR_VERSION
|
||||
unset DIB_HDFS_LIB_DIR
|
||||
unset DIB_MIN_TMPFS
|
||||
unset plugin_type
|
||||
|
|
|
@ -28,3 +28,7 @@ DIB_CDH_HDFS_ONLY
|
|||
:Description: If set will install only the namenode and datanode
|
||||
packages with their dependencies.
|
||||
|
||||
DIB_CDH_MINOR_VERSION
|
||||
:Required: No
|
||||
:Description: If set will install minor version of CDH. Available minor
|
||||
versions are 5.7.x.
|
||||
|
|
|
@ -55,6 +55,11 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
case "$DIB_CDH_VERSION" in
|
||||
5.7)
|
||||
HADOOP_OPENSTACK_5_7_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar"
|
||||
;;
|
||||
esac
|
||||
HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar"
|
||||
HADOOP_OPENSTACK_5_4_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.4.0/hadoop-openstack-2.6.0-cdh5.4.0.jar"
|
||||
HADOOP_OPENSTACK_5_3_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.5.0-cdh5.3.0/hadoop-openstack-2.5.0-cdh5.3.0.jar"
|
||||
|
@ -79,10 +84,21 @@ case "$DIB_CDH_VERSION" in
|
|||
;;
|
||||
5.5)
|
||||
wget -O $dest $HADOOP_OPENSTACK_5_5_0_URL
|
||||
# Create links to keep backward version support.
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
|
||||
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
|
||||
# Create links to keep backward version support.
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
|
||||
fi
|
||||
;;
|
||||
5.7)
|
||||
wget -O $dest $HADOOP_OPENSTACK_5_7_x_URL
|
||||
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
|
||||
# Create links to keep backward version support.
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Unhandled version $DIB_CDH_VERSION for hadoop-openstack.jar"
|
||||
|
|
|
@ -156,22 +156,88 @@ function add_repo_5_5_0 {
|
|||
apt-get update
|
||||
;;
|
||||
centos | rhel | centos7 | rhel7 )
|
||||
case $DISTRO_NAME in
|
||||
centos | rhel )
|
||||
releasever=6
|
||||
;;
|
||||
centos7 | rhel7 )
|
||||
releasever=7
|
||||
;;
|
||||
esac
|
||||
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/5.5.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/5.5.0/" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
|
||||
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/5.5.0/' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/5.5.0/" >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
|
||||
echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo
|
||||
echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo
|
||||
echo 'baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/5.5.0/' >> /etc/yum.repos.d/kms.repo
|
||||
echo 'gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/kms.repo
|
||||
echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/5.5.0/" >> /etc/yum.repos.d/kms.repo
|
||||
echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo
|
||||
|
||||
yum clean all
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function add_repo_5_7_x {
|
||||
case $DISTRO_NAME in
|
||||
ubuntu )
|
||||
# Add repository with postgresql package (it's dependency of cloudera packages)
|
||||
# Base image doesn't contain this repo
|
||||
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list
|
||||
|
||||
# Cloudera repositories
|
||||
echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list
|
||||
echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list
|
||||
|
||||
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add -
|
||||
|
||||
echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list
|
||||
echo "deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list
|
||||
|
||||
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add -
|
||||
|
||||
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list
|
||||
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add -
|
||||
|
||||
#change repository priority
|
||||
echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin
|
||||
|
||||
apt-get update
|
||||
;;
|
||||
centos | rhel | centos7 | rhel7 )
|
||||
case $DISTRO_NAME in
|
||||
centos | rhel )
|
||||
releasever=6
|
||||
;;
|
||||
centos7 | rhel7 )
|
||||
releasever=7
|
||||
;;
|
||||
esac
|
||||
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
|
||||
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
|
||||
echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo
|
||||
echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo
|
||||
echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo
|
||||
echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo
|
||||
|
||||
yum clean all
|
||||
|
@ -197,6 +263,10 @@ case "$DIB_CDH_VERSION" in
|
|||
echo "Installing CDH Version $DIB_CDH_VERSION..."
|
||||
add_repo_5_5_0
|
||||
;;
|
||||
5.7)
|
||||
echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..."
|
||||
add_repo_5_7_x
|
||||
;;
|
||||
*)
|
||||
echo "Invalid CDH Version : $DIB_CDH_VERSION specified, exiting install."
|
||||
exit 1
|
||||
|
|
|
@ -25,12 +25,12 @@ case "$plugin_type" in
|
|||
hadoop="2.3.0";;
|
||||
"5.3")
|
||||
hadoop="2.5.0";;
|
||||
"5.4" | "5.5")
|
||||
"5.4" | "5.5" | "5.7")
|
||||
hadoop="2.6.0";;
|
||||
esac
|
||||
case "$plugin_type" in
|
||||
"cloudera")
|
||||
DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop-cdh$DIB_CDH_VERSION.0.jar";;
|
||||
DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop-cdh$DIB_CDH_MINOR_VERSION.jar";;
|
||||
"spark")
|
||||
DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack.jar";;
|
||||
esac
|
||||
|
|
|
@ -7,19 +7,16 @@ case "$PLUGIN" in
|
|||
# manually call each distro for cloudera, so we can cleanup
|
||||
# cache and generated images from the previous run
|
||||
# (saving space a bit)
|
||||
for distro in ubuntu centos; do
|
||||
for distro in ubuntu centos centos7; do
|
||||
# NOTE(vgridnev): Number of versions are growing; testing only 2 latest
|
||||
# releases of CDH
|
||||
for version in 5.4 5.5; do
|
||||
# NOTE(mionkin) centos7 is available only for CDH 5.5 and 5.7
|
||||
for version in 5.5 5.7; do
|
||||
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version
|
||||
find . -maxdepth 1 -name '*cloudera*.qcow2' -delete
|
||||
sudo rm -rf ~/.cache/image-create
|
||||
done
|
||||
done
|
||||
# NOTE(mionkin) centos7 is available only for cloudera 5.5
|
||||
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i centos7 -v 5.5
|
||||
find . -maxdepth 1 -name '*cloudera*.qcow2' -delete
|
||||
sudo rm -rf ~/.cache/image-create
|
||||
;;
|
||||
*)
|
||||
tox -e venv -- sahara-image-create -x -u -p $PLUGIN
|
||||
|
|
Loading…
Reference in New Issue