openstack/sahara-image-elements
Code Issues Proposed changes

Remove some codes of older version builder

Browse Source 
Building CDH image under version 5.5.0 is no longer support.
Remove these useless code.
Also, adding the ambari usage info in sahara-image-create
command.

Change-Id: I6fffe25ee9daf651355611be675137babb67e2a8
This commit is contained in:
Shu Yingya 2017-04-12 20:32:40 +08:00
parent 2113ac4861
commit 20126fbde6
6 changed files with 5 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
diskimage-create/README.rst
View File

@ -17,7 +17,7 @@ Environment variables the script accepts are 'DIB_HADOOP_VERSION_2_7_1',
'[ubuntu|fedora|centos|centos7]_vanilla_hadoop_2_7_1_image_name',
'ubuntu_spark_image_name', 'ubuntu_storm_image_name',
'ambari_[ubuntu|centos|centos7]_image_name',
'cloudera_[5_0|5_3|5_4|5_5|5_7|5_9]_[ubuntu|centos]_image_name',
'cloudera_[5_5|5_7|5_9]_[ubuntu|centos]_image_name',
'mapr_[ubuntu|centos|centos7]_image_name',
'plain_[ubuntu|fedora|centos|centos7]_image_name'.
@ -59,14 +59,14 @@ like this:
.. sourcecode:: bash
tox -e venv -- sahara-image-create -v [2.7.1|4|5.0|5.3|5.4|5.5|5.7|5.9]
tox -e venv -- sahara-image-create -v [2.7.1|5.5|5.7|5.9]
Also, if you are planning to select which ambari version to target use the
'-v' commandline option like this:
.. sourcecode:: bash
tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0]
tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0|2.4.2.0]
6. To select which operating system to target use the '-i' commandline option
like this:

53
diskimage-create/diskimage-create.sh
View File

@ -32,7 +32,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 2.7.1|5.5|5.7|5.9|2.2.0.0|2.2.1.0]"
echo " [-v 2.7.1|5.5|5.7|5.9|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-r 5.1.0|5.2.0]"
echo " [-s 1.3.1|1.6.0]"
echo " [-t 0.9.2|1.0.1]"
@ -651,33 +651,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
cloudera_elements_sequence="hadoop-cloudera swift_hadoop kdc"
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
export DIB_CDH_VERSION="5.0"
cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0}
# Cloudera supports only 12.04 Ubuntu
export DIB_RELEASE="precise"
image_create ubuntu $cloudera_5_0_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
export DIB_CDH_VERSION="5.3"
cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0}
# Cloudera supports only 12.04 Ubuntu
export DIB_RELEASE="precise"
image_create ubuntu $cloudera_5_3_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.4" ]; then
export DIB_CDH_VERSION="5.4"
cloudera_5_4_ubuntu_image_name=${cloudera_5_4_ubuntu_image_name:-ubuntu_sahara_cloudera_5_4_0}
# Cloudera supports only 12.04 Ubuntu
export DIB_RELEASE="precise"
image_create ubuntu $cloudera_5_4_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then
export DIB_CDH_VERSION="5.5"
cloudera_5_5_ubuntu_image_name=${cloudera_5_5_ubuntu_image_name:-ubuntu_sahara_cloudera_5_5_0}
@ -709,30 +682,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then
centos_cloudera_elements_sequence="selinux-permissive disable-firewall nc"
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
export DIB_CDH_VERSION="5.0"
cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0}
image_create centos $cloudera_5_0_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence
unset DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
export DIB_CDH_VERSION="5.3"
cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0}
image_create centos $cloudera_5_3_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence
unset DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.4" ]; then
export DIB_CDH_VERSION="5.4"
cloudera_5_4_centos_image_name=${cloudera_5_4_centos_image_name:-centos_sahara_cloudera_5_4_0}
image_create centos $cloudera_5_4_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence
unset DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then
export DIB_CDH_VERSION="5.5"

26
elements/hadoop-cloudera/install.d/50-install-cloudera
View File

@ -34,7 +34,6 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
zookeeper
if [ $DIB_CDH_VERSION \> "5.0" ]; then
# CDH5.0 does not have below packages.
install-packages \
flume-ng \
hadoop-kms \
@ -50,12 +49,7 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
solr-doc \
search \
spark-history-server \
sqoop2
fi
if [ $DIB_CDH_VERSION \> "5.4" ]; then
# we support Kafka from CDH 5.5
install-packages \
sqoop2 \
kafka \
kafka-server
fi
@ -72,27 +66,9 @@ case "$DIB_CDH_VERSION" in
;;
esac
HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar"
HADOOP_OPENSTACK_5_4_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.4.0/hadoop-openstack-2.6.0-cdh5.4.0.jar"
HADOOP_OPENSTACK_5_3_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.5.0-cdh5.3.0/hadoop-openstack-2.5.0-cdh5.3.0.jar"
HADOOP_OPENSTACK_5_0_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.3.0-cdh5.0.0/hadoop-openstack-2.3.0-cdh5.0.0.jar"
dest=/usr/lib/hadoop-mapreduce/hadoop-openstack.jar
case "$DIB_CDH_VERSION" in
5.0)
wget -O $dest $HADOOP_OPENSTACK_5_0_0_URL
;;
5.3)
wget -O $dest $HADOOP_OPENSTACK_5_3_0_URL
;;
5.4)
wget -O $dest $HADOOP_OPENSTACK_5_4_0_URL
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
# Create links to keep backward version support.
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
fi
;;
5.5)
wget -O $dest $HADOOP_OPENSTACK_5_5_0_URL
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then

135
elements/hadoop-cloudera/pre-install.d/10-add-mirror
View File

@ -6,129 +6,6 @@ fi
set -eu
set -o pipefail
function add_repo_5_0_0 {
case $DISTRO_NAME in
ubuntu )
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
apt-get update
;;
centos | rhel )
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
yum clean all
;;
esac
}
function add_repo_5_3_0 {
case $DISTRO_NAME in
ubuntu )
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/cloudera.list
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/archive.key | apt-key add -
apt-get update
;;
centos | rhel )
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
wget -O /etc/yum.repos.d/kms.repo http://archive.cloudera.com/navigator-keytrustee5/redhat/6/x86_64/navigator-keytrustee/navigator-keytrustee5.repo
yum clean all
;;
esac
}
function add_repo_5_4_0 {
case $DISTRO_NAME in
ubuntu )
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/cloudera.list
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/archive.key | apt-key add -
apt-get update
;;
centos | rhel )
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.4.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.4.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
wget -O /etc/yum.repos.d/kms.repo http://archive.cloudera.com/navigator-keytrustee5/redhat/6/x86_64/navigator-keytrustee/navigator-keytrustee5.repo
yum clean all
;;
esac
}
function add_repo_5_5_0 {
case $DISTRO_NAME in
ubuntu )
@ -337,18 +214,6 @@ DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0}
# Call version-specific script to install the desired version of CDH
case "$DIB_CDH_VERSION" in
5.0)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_0_0
;;
5.3)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_3_0
;;
5.4)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_4_0
;;
5.5)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_5_0

9
elements/spark/root.d/50-download-spark
View File

@ -15,18 +15,9 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
# INFO on hadoop versions: http://spark.apache.org/docs/latest/hadoop-third-party-distributions.html
if [ -z "${SPARK_HADOOP_DL:-}" ]; then
case "${DIB_CDH_VERSION:-}" in
5.0)
SPARK_HADOOP_DL=hadoop2.3
;;
5.3)
SPARK_HADOOP_DL=hadoop2.4
;;
5.4)
SPARK_HADOOP_DL=hadoop2.6
;;
CDH4)
SPARK_HADOOP_DL=cdh4
;;
*)
echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported."
echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version."

6
elements/swift_hadoop/post-install.d/81-add-jar
View File

@ -17,12 +17,6 @@ case "$plugin_type" in
;;
"cloudera" | "spark")
case "$DIB_CDH_VERSION" in
"CDH4")
hadoop="2.2.0";;
"5.0")
hadoop="2.3.0";;
"5.3")
hadoop="2.5.0";;
"5.4" | "5.5" | "5.7" | "5.9")
hadoop="2.6.0";;
esac